aws_recon 0.2.4 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/stale.yml +17 -0
- data/.gitignore +1 -0
- data/Dockerfile +34 -0
- data/LICENSE.txt +1 -1
- data/aws_recon.gemspec +1 -1
- data/binstub/aws_recon +10 -0
- data/lib/aws_recon/aws_recon.rb +1 -1
- data/lib/aws_recon/collectors/ec2.rb +17 -0
- data/lib/aws_recon/collectors/elasticache.rb +22 -0
- data/lib/aws_recon/collectors/iam.rb +42 -1
- data/lib/aws_recon/collectors/shield.rb +2 -2
- data/lib/aws_recon/lib/formatter.rb +1 -1
- data/lib/aws_recon/lib/mapper.rb +2 -1
- data/lib/aws_recon/options.rb +7 -0
- data/lib/aws_recon/services.yaml +2 -2
- data/lib/aws_recon/version.rb +1 -1
- data/readme.md +56 -5
- metadata +10 -7
- data/Gemfile.lock +0 -1000
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 748e27233aa80e92e6f74f5685c9620315a649cc425c96186d234650b7c56242
|
4
|
+
data.tar.gz: ebb4fd703ffa348040d6659b9095c5a8c5ac49e60707f08958a84f43fa46440c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8bf432c66917846ca2982d566b570d8b1930dff31168847732132ca033c679c9595b2c0729e6d48da13814ef84e76ec4809af288ebb6097e27183b224d8cf30e
|
7
|
+
data.tar.gz: cd2694d0a363bf37ad38e0ec1c3d0f1dfd9488e85541169ba2cf3c56a078ce9088406e4f58d8dee2b3b87240969532d8dfaf87fa666413cf2c55da69eeeaea9d
|
data/.github/stale.yml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# Number of days of inactivity before an issue becomes stale
|
2
|
+
daysUntilStale: 30
|
3
|
+
# Number of days of inactivity before a stale issue is closed
|
4
|
+
daysUntilClose: 5
|
5
|
+
# Issues with these labels will never be considered stale
|
6
|
+
exemptLabels:
|
7
|
+
- pinned
|
8
|
+
- security
|
9
|
+
# Label to use when marking an issue as stale
|
10
|
+
staleLabel: wontfix
|
11
|
+
# Comment to post when marking an issue as stale. Set to `false` to disable
|
12
|
+
markComment: >
|
13
|
+
This issue has been automatically marked as stale because it has not had
|
14
|
+
recent activity. It will be closed if no further activity occurs. Thank you
|
15
|
+
for your contributions.
|
16
|
+
# Comment to post when closing a stale issue. Set to `false` to disable
|
17
|
+
closeComment: false
|
data/.gitignore
CHANGED
data/Dockerfile
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
ARG RUBY_VERSION=2.6.6
|
2
|
+
FROM ruby:${RUBY_VERSION}-alpine
|
3
|
+
|
4
|
+
LABEL maintainer="Darkbit <info@darkbit.io>"
|
5
|
+
|
6
|
+
ARG USER=recon
|
7
|
+
ARG GEM=aws_recon
|
8
|
+
ARG VERSION=0.2.8
|
9
|
+
ARG BUNDLER_VERSION=2.1.4
|
10
|
+
|
11
|
+
# Install new Bundler version
|
12
|
+
RUN rm /usr/local/lib/ruby/gems/*/specifications/default/bundler-*.gemspec && \
|
13
|
+
gem uninstall bundler && \
|
14
|
+
gem install bundler -v ${BUNDLER_VERSION}
|
15
|
+
|
16
|
+
# Install gem
|
17
|
+
RUN gem install ${GEM} -v ${VERSION}
|
18
|
+
|
19
|
+
# Create non-root user
|
20
|
+
RUN addgroup -S ${USER} && \
|
21
|
+
adduser -S ${USER} \
|
22
|
+
-G ${USER} \
|
23
|
+
-s /bin/ash \
|
24
|
+
-h /${USER}
|
25
|
+
|
26
|
+
# Copy binstub
|
27
|
+
COPY binstub/${GEM} /usr/local/bundle/bin/
|
28
|
+
RUN chmod +x /usr/local/bundle/bin/${GEM}
|
29
|
+
|
30
|
+
# Switch user
|
31
|
+
USER ${USER}
|
32
|
+
WORKDIR /${USER}
|
33
|
+
|
34
|
+
CMD ["ash"]
|
data/LICENSE.txt
CHANGED
data/aws_recon.gemspec
CHANGED
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
|
|
28
28
|
|
29
29
|
spec.add_development_dependency 'bundler', '~> 1.17'
|
30
30
|
spec.add_development_dependency 'gem-release', '~> 2.1'
|
31
|
-
spec.add_development_dependency 'rake', '
|
31
|
+
spec.add_development_dependency 'rake', '~> 12.3'
|
32
32
|
spec.add_development_dependency 'minitest', '~> 5.0'
|
33
33
|
spec.add_development_dependency 'solargraph', '~> 0.39.11'
|
34
34
|
spec.add_development_dependency 'rubocop', '~> 0.87.1'
|
data/binstub/aws_recon
ADDED
data/lib/aws_recon/aws_recon.rb
CHANGED
@@ -44,7 +44,7 @@ module AwsRecon
|
|
44
44
|
#
|
45
45
|
def collect(service, region)
|
46
46
|
mapper = Object.const_get(service.name)
|
47
|
-
resources = mapper.new(service.name, region, @options)
|
47
|
+
resources = mapper.new(@account_id, service.name, region, @options)
|
48
48
|
|
49
49
|
collection = resources.collect.map do |resource|
|
50
50
|
if @options.output_format == 'custom'
|
@@ -48,6 +48,23 @@ class EC2 < Mapper
|
|
48
48
|
struct.arn = instance.instance_id # no true ARN
|
49
49
|
struct.reservation_id = reservation.reservation_id
|
50
50
|
|
51
|
+
# collect instance user_data
|
52
|
+
if @options.collect_user_data
|
53
|
+
user_data_raw = @client.describe_instance_attribute({
|
54
|
+
attribute: 'userData',
|
55
|
+
instance_id: instance.instance_id
|
56
|
+
}).user_data.to_h[:value]
|
57
|
+
|
58
|
+
# don't save non-string user_data
|
59
|
+
if user_data_raw
|
60
|
+
user_data = Base64.decode64(user_data_raw)
|
61
|
+
|
62
|
+
if user_data.force_encoding('UTF-8').ascii_only?
|
63
|
+
struct.user_data = user_data
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
51
68
|
resources.push(struct.to_h)
|
52
69
|
end
|
53
70
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class ElastiCache < Mapper
|
2
|
+
def collect
|
3
|
+
resources = []
|
4
|
+
|
5
|
+
#
|
6
|
+
# describe_cache_clusters
|
7
|
+
#
|
8
|
+
@client.describe_cache_clusters.each_with_index do |response, page|
|
9
|
+
log(response.context.operation_name, page)
|
10
|
+
|
11
|
+
response.cache_clusters.each do |cluster|
|
12
|
+
struct = OpenStruct.new(cluster.to_h)
|
13
|
+
struct.type = 'cluster'
|
14
|
+
struct.arn = cluster.arn
|
15
|
+
|
16
|
+
resources.push(struct.to_h)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
resources
|
21
|
+
end
|
22
|
+
end
|
@@ -10,7 +10,10 @@ class IAM < Mapper
|
|
10
10
|
# list_mfa_devices
|
11
11
|
# list_ssh_public_keys
|
12
12
|
#
|
13
|
-
|
13
|
+
opts = {
|
14
|
+
filter: %w[User Role Group LocalManagedPolicy AWSManagedPolicy]
|
15
|
+
}
|
16
|
+
@client.get_account_authorization_details(opts).each_with_index do |response, page|
|
14
17
|
log(response.context.operation_name, page)
|
15
18
|
|
16
19
|
# users
|
@@ -19,6 +22,14 @@ class IAM < Mapper
|
|
19
22
|
struct.type = 'user'
|
20
23
|
struct.mfa_devices = @client.list_mfa_devices({ user_name: user.user_name }).mfa_devices.map(&:to_h)
|
21
24
|
struct.ssh_keys = @client.list_ssh_public_keys({ user_name: user.user_name }).ssh_public_keys.map(&:to_h)
|
25
|
+
struct.user_policy_list = if user.user_policy_list
|
26
|
+
user.user_policy_list.map do |p|
|
27
|
+
{
|
28
|
+
policy_name: p.policy_name,
|
29
|
+
policy_document: JSON.parse(CGI.unescape(p.policy_document))
|
30
|
+
}
|
31
|
+
end
|
32
|
+
end
|
22
33
|
|
23
34
|
resources.push(struct.to_h)
|
24
35
|
end
|
@@ -27,6 +38,14 @@ class IAM < Mapper
|
|
27
38
|
response.group_detail_list.each do |group|
|
28
39
|
struct = OpenStruct.new(group.to_h)
|
29
40
|
struct.type = 'group'
|
41
|
+
struct.group_policy_list = if group.group_policy_list
|
42
|
+
group.group_policy_list.map do |p|
|
43
|
+
{
|
44
|
+
policy_name: p.policy_name,
|
45
|
+
policy_document: JSON.parse(CGI.unescape(p.policy_document))
|
46
|
+
}
|
47
|
+
end
|
48
|
+
end
|
30
49
|
|
31
50
|
resources.push(struct.to_h)
|
32
51
|
end
|
@@ -35,6 +54,15 @@ class IAM < Mapper
|
|
35
54
|
response.role_detail_list.each do |role|
|
36
55
|
struct = OpenStruct.new(role.to_h)
|
37
56
|
struct.type = 'role'
|
57
|
+
struct.assume_role_policy_document = JSON.parse(CGI.unescape(role.assume_role_policy_document))
|
58
|
+
struct.role_policy_list = if role.role_policy_list
|
59
|
+
role.role_policy_list.map do |p|
|
60
|
+
{
|
61
|
+
policy_name: p.policy_name,
|
62
|
+
policy_document: JSON.parse(CGI.unescape(p.policy_document))
|
63
|
+
}
|
64
|
+
end
|
65
|
+
end
|
38
66
|
|
39
67
|
resources.push(struct.to_h)
|
40
68
|
end
|
@@ -43,6 +71,16 @@ class IAM < Mapper
|
|
43
71
|
response.policies.each do |policy|
|
44
72
|
struct = OpenStruct.new(policy.to_h)
|
45
73
|
struct.type = 'policy'
|
74
|
+
struct.policy_version_list = if policy.policy_version_list
|
75
|
+
policy.policy_version_list.map do |p|
|
76
|
+
{
|
77
|
+
version_id: p.version_id,
|
78
|
+
document: JSON.parse(CGI.unescape(p.document)),
|
79
|
+
is_default_version: p.is_default_version,
|
80
|
+
create_date: p.create_date
|
81
|
+
}
|
82
|
+
end
|
83
|
+
end
|
46
84
|
|
47
85
|
resources.push(struct.to_h)
|
48
86
|
end
|
@@ -56,6 +94,7 @@ class IAM < Mapper
|
|
56
94
|
|
57
95
|
struct = OpenStruct.new(response.password_policy.to_h)
|
58
96
|
struct.type = 'password_policy'
|
97
|
+
struct.arn = "arn:aws:iam::#{@account}:account_password_policy/global"
|
59
98
|
|
60
99
|
resources.push(struct.to_h)
|
61
100
|
end
|
@@ -68,6 +107,7 @@ class IAM < Mapper
|
|
68
107
|
|
69
108
|
struct = OpenStruct.new(response.summary_map)
|
70
109
|
struct.type = 'account_summary'
|
110
|
+
struct.arn = "arn:aws:iam::#{@account}:account_summary/global"
|
71
111
|
|
72
112
|
resources.push(struct.to_h)
|
73
113
|
end
|
@@ -111,6 +151,7 @@ class IAM < Mapper
|
|
111
151
|
|
112
152
|
struct = OpenStruct.new
|
113
153
|
struct.type = 'credential_report'
|
154
|
+
struct.arn = "arn:aws:iam::#{@account}:credential_report/global"
|
114
155
|
struct.content = CSV.parse(response.content, headers: :first_row).map(&:to_h)
|
115
156
|
struct.report_format = response.report_format
|
116
157
|
struct.generated_time = response.generated_time
|
@@ -13,7 +13,7 @@ class Shield < Mapper
|
|
13
13
|
|
14
14
|
struct = OpenStruct.new(response.subscription.to_h)
|
15
15
|
struct.type = 'subscription'
|
16
|
-
struct.arn = "arn:aws:shield:#{@region}:#{account}:subscription"
|
16
|
+
struct.arn = "arn:aws:shield:#{@region}:#{@account}:subscription"
|
17
17
|
|
18
18
|
resources.push(struct.to_h)
|
19
19
|
end
|
@@ -26,7 +26,7 @@ class Shield < Mapper
|
|
26
26
|
|
27
27
|
struct = OpenStruct.new
|
28
28
|
struct.type = 'contact_list'
|
29
|
-
struct.arn = "arn:aws:shield:#{@region}:#{account}:contact_list"
|
29
|
+
struct.arn = "arn:aws:shield:#{@region}:#{@account}:contact_list"
|
30
30
|
struct.contacts = response.emergency_contact_list.map(&:to_h)
|
31
31
|
|
32
32
|
resources.push(struct.to_h)
|
@@ -8,7 +8,7 @@ class Formatter
|
|
8
8
|
def custom(account_id, region, service, resource)
|
9
9
|
{
|
10
10
|
account: account_id,
|
11
|
-
name: resource[:arn]
|
11
|
+
name: resource[:arn],
|
12
12
|
service: service.name,
|
13
13
|
region: region,
|
14
14
|
asset_type: resource[:type],
|
data/lib/aws_recon/lib/mapper.rb
CHANGED
@@ -22,7 +22,8 @@ class Mapper
|
|
22
22
|
# S3 (unless the bucket was created in another region)
|
23
23
|
SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
|
24
24
|
|
25
|
-
def initialize(service, region, options)
|
25
|
+
def initialize(account, service, region, options)
|
26
|
+
@account = account
|
26
27
|
@service = service
|
27
28
|
@region = region
|
28
29
|
@options = options
|
data/lib/aws_recon/options.rb
CHANGED
@@ -15,6 +15,7 @@ class Parser
|
|
15
15
|
:output_file,
|
16
16
|
:output_format,
|
17
17
|
:threads,
|
18
|
+
:collect_user_data,
|
18
19
|
:skip_slow,
|
19
20
|
:stream_output,
|
20
21
|
:verbose,
|
@@ -43,6 +44,7 @@ class Parser
|
|
43
44
|
false,
|
44
45
|
false,
|
45
46
|
false,
|
47
|
+
false,
|
46
48
|
false
|
47
49
|
)
|
48
50
|
|
@@ -103,6 +105,11 @@ class Parser
|
|
103
105
|
end
|
104
106
|
end
|
105
107
|
|
108
|
+
# collect EC2 instance user data
|
109
|
+
opts.on('-u', '--user-data', 'Collect EC2 instance user data (default: false)') do
|
110
|
+
args.collect_user_data = true
|
111
|
+
end
|
112
|
+
|
106
113
|
# skip slow operations
|
107
114
|
opts.on('-z', '--skip-slow', 'Skip slow operations (default: false)') do
|
108
115
|
args.skip_slow = true
|
data/lib/aws_recon/services.yaml
CHANGED
@@ -21,8 +21,6 @@
|
|
21
21
|
alias: ec2
|
22
22
|
- name: EKS
|
23
23
|
alias: eks
|
24
|
-
excluded_regions:
|
25
|
-
- us-west-1
|
26
24
|
- name: ECS
|
27
25
|
alias: ecs
|
28
26
|
- name: ElasticLoadBalancing
|
@@ -33,6 +31,8 @@
|
|
33
31
|
alias: elbv2
|
34
32
|
excluded_regions:
|
35
33
|
- ap-southeast-1
|
34
|
+
- name: ElastiCache
|
35
|
+
alias: elasticache
|
36
36
|
- name: IAM
|
37
37
|
global: true
|
38
38
|
alias: iam
|
data/lib/aws_recon/version.rb
CHANGED
data/readme.md
CHANGED
@@ -26,18 +26,20 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
|
|
26
26
|
|
27
27
|
### Installation
|
28
28
|
|
29
|
-
|
29
|
+
AWS Recon can be run locally by installing the Ruby gem, or via a Docker container.
|
30
|
+
|
31
|
+
To run locally, first install the gem:
|
30
32
|
|
31
33
|
```
|
32
34
|
$ gem install aws_recon
|
33
|
-
Fetching aws_recon-0.2.
|
35
|
+
Fetching aws_recon-0.2.8.gem
|
34
36
|
Fetching aws-sdk-resources-3.76.0.gem
|
35
37
|
Fetching aws-sdk-3.0.1.gem
|
36
38
|
Fetching parallel-1.19.2.gem
|
37
39
|
...
|
38
40
|
Successfully installed aws-sdk-3.0.1
|
39
41
|
Successfully installed parallel-1.19.2
|
40
|
-
Successfully installed aws_recon-0.2.
|
42
|
+
Successfully installed aws_recon-0.2.8
|
41
43
|
```
|
42
44
|
|
43
45
|
Or add it to your Gemfile using `bundle`:
|
@@ -49,9 +51,23 @@ Resolving dependencies...
|
|
49
51
|
...
|
50
52
|
Using aws-sdk 3.0.1
|
51
53
|
Using parallel 1.19.2
|
52
|
-
Using aws_recon 0.2.
|
54
|
+
Using aws_recon 0.2.8
|
55
|
+
```
|
56
|
+
|
57
|
+
To run via a Docker a container, pass the necessary AWS credentials into the Docker `run` command. For example:
|
58
|
+
|
59
|
+
```
|
60
|
+
$ docker run -t --rm \
|
61
|
+
-e AWS_REGION \
|
62
|
+
-e AWS_ACCESS_KEY_ID \
|
63
|
+
-e AWS_SECRET_ACCESS_KEY \
|
64
|
+
-e AWS_SESSION_TOKEN \
|
65
|
+
-v $(pwd)/output.json:/recon/output.json \
|
66
|
+
darkbitio/aws_recon:latest \
|
67
|
+
aws_recon -v -s EC2 -r global,us-east-1,us-east-2
|
53
68
|
```
|
54
69
|
|
70
|
+
|
55
71
|
## Usage
|
56
72
|
|
57
73
|
AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials.
|
@@ -66,6 +82,39 @@ Plain environment variables will work fine too.
|
|
66
82
|
$ AWS_PROFILE=<profile> aws_recon
|
67
83
|
```
|
68
84
|
|
85
|
+
To run from a Docker container using `aws-vault` managed credentials (output to stdout):
|
86
|
+
|
87
|
+
```
|
88
|
+
$ aws-vault exec <vault_profile> -- docker run -t --rm \
|
89
|
+
-e AWS_REGION \
|
90
|
+
-e AWS_ACCESS_KEY_ID \
|
91
|
+
-e AWS_SECRET_ACCESS_KEY \
|
92
|
+
-e AWS_SESSION_TOKEN \
|
93
|
+
darkbitio/aws_recon:latest \
|
94
|
+
aws_recon -j -s EC2 -r global,us-east-1,us-east-2
|
95
|
+
```
|
96
|
+
|
97
|
+
To run from a Docker container using `aws-vault` managed credentials and output to a file, you will need to satisfy a couple of requirements. First, Docker needs access to bind mount the path you specify (or a parent path above). Second, you need to create an empty file to save the output into (e.g. `output.json`). This is because we are only mounting that one file into the Docker container at run time. For example:
|
98
|
+
|
99
|
+
Create an empty file.
|
100
|
+
|
101
|
+
```
|
102
|
+
$ touch output.json
|
103
|
+
```
|
104
|
+
|
105
|
+
Run the `aws_recon` container, specifying the output file.
|
106
|
+
|
107
|
+
```
|
108
|
+
$ aws-vault exec <vault_profile> -- docker run -t --rm \
|
109
|
+
-e AWS_REGION \
|
110
|
+
-e AWS_ACCESS_KEY_ID \
|
111
|
+
-e AWS_SECRET_ACCESS_KEY \
|
112
|
+
-e AWS_SESSION_TOKEN \
|
113
|
+
-v $(pwd)/output.json:/recon/output.json \
|
114
|
+
darkbitio/aws_recon:latest \
|
115
|
+
aws_recon -s EC2 -v -r global,us-east-1,us-east-2
|
116
|
+
```
|
117
|
+
|
69
118
|
You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running.
|
70
119
|
|
71
120
|
In verbose mode, the console output will show:
|
@@ -135,7 +184,7 @@ Most users will want to limit collection to relevant services and regions. Runni
|
|
135
184
|
```
|
136
185
|
$ aws_recon -h
|
137
186
|
|
138
|
-
AWS Recon - AWS Inventory Collector
|
187
|
+
AWS Recon - AWS Inventory Collector (0.2.8)
|
139
188
|
|
140
189
|
Usage: aws_recon [options]
|
141
190
|
-r, --regions [REGIONS] Regions to scan, separated by comma (default: all)
|
@@ -146,6 +195,7 @@ Usage: aws_recon [options]
|
|
146
195
|
-o, --output [OUTPUT] Specify output file (default: output.json)
|
147
196
|
-f, --format [FORMAT] Specify output format (default: aws)
|
148
197
|
-t, --threads [THREADS] Specify max threads (default: 8, max: 128)
|
198
|
+
-u, --user-data Collect EC2 instance user data (default: false)
|
149
199
|
-z, --skip-slow Skip slow operations (default: false)
|
150
200
|
-j, --stream-output Stream JSON lines to stdout (default: false)
|
151
201
|
-v, --verbose Output client progress and current operation
|
@@ -193,6 +243,7 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
|
|
193
243
|
- [x] ELB
|
194
244
|
- [x] EKS
|
195
245
|
- [x] Elasticsearch
|
246
|
+
- [x] ElastiCache
|
196
247
|
- [x] Firehose
|
197
248
|
- [ ] FMS
|
198
249
|
- [ ] Glacier
|