aws_recon 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/aws_recon.gemspec +4 -4
- data/bin/aws_recon +3 -0
- data/lib/aws_recon/aws_recon.rb +1 -1
- data/lib/aws_recon/collectors/s3.rb +8 -6
- data/lib/aws_recon/lib/mapper.rb +9 -2
- data/lib/aws_recon/options.rb +1 -1
- data/lib/aws_recon/version.rb +1 -1
- data/readme.md +44 -14
- metadata +11 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ac6fae11753e715682d656eba8c922267831fe7324d9a44bd96296543cf9653e
|
4
|
+
data.tar.gz: 900bf49cc999b1fd9d227067609ad85c3b6a462886156c94e7dfb5d5e1c0a982
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6c76cf3f96cbf58501c61861361be27e76cc95e1c60cf83ae141d2e4aa0a4d12efe7ed9b1dae74094fff2a5a157a2d17a49ef37ecc471de7f985156b4228e608
|
7
|
+
data.tar.gz: 93341c804a9daf7c4f849927ff429a34b72af74acdbe741e324829e938d01aa805550e7a41b0aeeeb68c2dc80ec37e14cb83e7c5ef1f9406c7c94815eca6757e
|
data/Gemfile.lock
CHANGED
@@ -947,7 +947,7 @@ GEM
|
|
947
947
|
coderay (~> 1.1)
|
948
948
|
method_source (~> 1.0)
|
949
949
|
rainbow (3.0.0)
|
950
|
-
rake (
|
950
|
+
rake (13.0.1)
|
951
951
|
regexp_parser (1.7.1)
|
952
952
|
reverse_markdown (2.0.0)
|
953
953
|
nokogiri
|
@@ -992,7 +992,7 @@ DEPENDENCIES
|
|
992
992
|
gem-release (~> 2.1)
|
993
993
|
minitest (~> 5.0)
|
994
994
|
pry (~> 0.13.1)
|
995
|
-
rake (
|
995
|
+
rake (>= 12.3.3)
|
996
996
|
rubocop (~> 0.87.1)
|
997
997
|
solargraph (~> 0.39.11)
|
998
998
|
|
data/aws_recon.gemspec
CHANGED
@@ -7,10 +7,10 @@ require 'aws_recon/version'
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
8
|
spec.name = 'aws_recon'
|
9
9
|
spec.version = AwsRecon::VERSION
|
10
|
-
spec.authors = ['Josh Larsen']
|
10
|
+
spec.authors = ['Josh Larsen', 'Darkbit']
|
11
11
|
spec.required_ruby_version = '>= 2.5.0'
|
12
|
-
spec.summary = 'A multi-threaded AWS inventory collection tool.'
|
13
|
-
spec.description =
|
12
|
+
spec.summary = 'A multi-threaded AWS inventory collection cli tool.'
|
13
|
+
spec.description = 'AWS Recon is a command line tool to collect resources from an Amazon Web Services (AWS) account. The tool outputs JSON suitable for processing with other tools.'
|
14
14
|
spec.homepage = 'https://github.com/darkbitio/aws-recon'
|
15
15
|
spec.license = 'MIT'
|
16
16
|
|
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
|
|
28
28
|
|
29
29
|
spec.add_development_dependency 'bundler', '~> 1.17'
|
30
30
|
spec.add_development_dependency 'gem-release', '~> 2.1'
|
31
|
-
spec.add_development_dependency 'rake', '
|
31
|
+
spec.add_development_dependency 'rake', '>= 12.3.3'
|
32
32
|
spec.add_development_dependency 'minitest', '~> 5.0'
|
33
33
|
spec.add_development_dependency 'solargraph', '~> 0.39.11'
|
34
34
|
spec.add_development_dependency 'rubocop', '~> 0.87.1'
|
data/bin/aws_recon
CHANGED
data/lib/aws_recon/aws_recon.rb
CHANGED
@@ -88,7 +88,7 @@ module AwsRecon
|
|
88
88
|
@regions.filter { |x| x != 'global' }.each do |region|
|
89
89
|
Parallel.map(@aws_services.map { |x| OpenStruct.new(x) }.filter { |s| !s.global }.each, in_threads: @options.threads) do |service|
|
90
90
|
# some services aren't available in some regions
|
91
|
-
skip_region =
|
91
|
+
skip_region = service&.excluded_regions&.include?(region)
|
92
92
|
|
93
93
|
# user included this region in the args
|
94
94
|
next unless @regions.include?(region) && !skip_region
|
@@ -15,8 +15,6 @@ class S3 < Mapper
|
|
15
15
|
log(response.context.operation_name, page)
|
16
16
|
|
17
17
|
Parallel.map(response.buckets.each, in_threads: @options.threads) do |bucket|
|
18
|
-
# use shared client instance
|
19
|
-
client = @client
|
20
18
|
@thread = Parallel.worker_number
|
21
19
|
log(response.context.operation_name, bucket.name)
|
22
20
|
|
@@ -27,10 +25,14 @@ class S3 < Mapper
|
|
27
25
|
# check bucket region constraint
|
28
26
|
location = @client.get_bucket_location({ bucket: bucket.name }).location_constraint
|
29
27
|
|
30
|
-
#
|
31
|
-
|
32
|
-
|
33
|
-
|
28
|
+
# if you use a region other than the us-east-1 endpoint
|
29
|
+
# to create a bucket, you must set the location_constraint
|
30
|
+
# bucket parameter to the same region. (https://docs.aws.amazon.com/general/latest/gr/s3.html)
|
31
|
+
client = if location.empty?
|
32
|
+
@client
|
33
|
+
else
|
34
|
+
Aws::S3::Client.new({ region: location })
|
35
|
+
end
|
34
36
|
|
35
37
|
operations = [
|
36
38
|
{ func: 'get_bucket_acl', key: 'acl', field: nil },
|
data/lib/aws_recon/lib/mapper.rb
CHANGED
@@ -15,6 +15,13 @@
|
|
15
15
|
# to add 5 seconds delay on each retry for a total max of 55 seconds.
|
16
16
|
#
|
17
17
|
class Mapper
|
18
|
+
# Services that use us-east-1 endpoint only:
|
19
|
+
# Organizations
|
20
|
+
# Route53Domains
|
21
|
+
# Shield
|
22
|
+
# S3 (unless the bucket was created in another region)
|
23
|
+
SINGLE_REGION_SERVICES = %w[route53domains s3 shield support organizations].freeze
|
24
|
+
|
18
25
|
def initialize(service, region, options)
|
19
26
|
@service = service
|
20
27
|
@region = region
|
@@ -39,8 +46,8 @@ class Mapper
|
|
39
46
|
# regional service
|
40
47
|
client_options.merge!({ region: region }) unless region == 'global'
|
41
48
|
|
42
|
-
#
|
43
|
-
client_options.merge!({ region: 'us-east-1' }) if service.downcase
|
49
|
+
# single region services
|
50
|
+
client_options.merge!({ region: 'us-east-1' }) if SINGLE_REGION_SERVICES.include?(service.downcase) # rubocop:disable Layout/LineLength
|
44
51
|
|
45
52
|
# debug with wire trace
|
46
53
|
client_options.merge!({ http_wire_trace: true }) if @options.debug
|
data/lib/aws_recon/options.rb
CHANGED
@@ -47,7 +47,7 @@ class Parser
|
|
47
47
|
)
|
48
48
|
|
49
49
|
opt_parser = OptionParser.new do |opts|
|
50
|
-
opts.banner = "\n\x1b[32mAWS Recon\x1b[0m - AWS Inventory Collector\n\nUsage: aws_recon [options]"
|
50
|
+
opts.banner = "\n\x1b[32mAWS Recon\x1b[0m - AWS Inventory Collector (#{AwsRecon::VERSION})\n\nUsage: aws_recon [options]"
|
51
51
|
|
52
52
|
# regions
|
53
53
|
opts.on('-r', '--regions [REGIONS]', 'Regions to scan, separated by comma (default: all)') do |regions|
|
data/lib/aws_recon/version.rb
CHANGED
data/readme.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
[![Gem Version](https://badge.fury.io/rb/aws_recon.svg)](https://badge.fury.io/rb/aws_recon)
|
2
|
+
|
1
3
|
# AWS Recon
|
2
4
|
|
3
5
|
A multi-threaded AWS inventory collection tool.
|
@@ -24,17 +26,30 @@ Ruby 2.5.x or 2.6.x (developed and tested with 2.6.5)
|
|
24
26
|
|
25
27
|
### Installation
|
26
28
|
|
27
|
-
|
29
|
+
Install the gem:
|
28
30
|
|
29
31
|
```
|
30
|
-
$
|
31
|
-
|
32
|
-
|
32
|
+
$ gem install aws_recon
|
33
|
+
Fetching aws_recon-0.2.2.gem
|
34
|
+
Fetching aws-sdk-resources-3.76.0.gem
|
35
|
+
Fetching aws-sdk-3.0.1.gem
|
36
|
+
Fetching parallel-1.19.2.gem
|
33
37
|
...
|
34
|
-
|
38
|
+
Successfully installed aws-sdk-3.0.1
|
39
|
+
Successfully installed parallel-1.19.2
|
40
|
+
Successfully installed aws_recon-0.2.2
|
41
|
+
```
|
42
|
+
|
43
|
+
Or add it to your Gemfile using `bundle`:
|
44
|
+
|
45
|
+
```
|
46
|
+
$ bundle add aws_recon
|
47
|
+
Fetching gem metadata from https://rubygems.org/
|
48
|
+
Resolving dependencies...
|
35
49
|
...
|
36
|
-
|
37
|
-
|
50
|
+
Using aws-sdk 3.0.1
|
51
|
+
Using parallel 1.19.2
|
52
|
+
Using aws_recon 0.2.2
|
38
53
|
```
|
39
54
|
|
40
55
|
## Usage
|
@@ -42,13 +57,13 @@ Use `bundle info [gemname]` to see where a bundled gem is installed.
|
|
42
57
|
AWS Recon will leverage any AWS credentials currently available to the environment it runs in. If you are collecting from multiple accounts, you may want to leverage something like [aws-vault](https://github.com/99designs/aws-vault) to manage different credentials.
|
43
58
|
|
44
59
|
```
|
45
|
-
$ aws-vault exec profile --
|
60
|
+
$ aws-vault exec profile -- aws_recon
|
46
61
|
```
|
47
62
|
|
48
63
|
Plain environment variables will work fine too.
|
49
64
|
|
50
65
|
```
|
51
|
-
$ AWS_PROFILE=<profile>
|
66
|
+
$ AWS_PROFILE=<profile> aws_recon
|
52
67
|
```
|
53
68
|
|
54
69
|
You may want to use the `-v` or `--verbose` flag initially to see status and activity while collection is running.
|
@@ -62,7 +77,7 @@ In verbose mode, the console output will show:
|
|
62
77
|
The `t` prefix indicates which thread a particular request is running under. Region, service, and operation indicate which request operation is currently in progress and where.
|
63
78
|
|
64
79
|
```
|
65
|
-
$
|
80
|
+
$ aws_recon -v
|
66
81
|
|
67
82
|
t0.global.EC2.describe_account_attributes
|
68
83
|
t2.global.S3.list_buckets
|
@@ -87,11 +102,11 @@ Finished in 46 seconds. Saving resources to output.json.
|
|
87
102
|
#### Example command line options
|
88
103
|
|
89
104
|
```
|
90
|
-
$ AWS_PROFILE=<profile>
|
105
|
+
$ AWS_PROFILE=<profile> aws_recon -s S3,EC2 -r global,us-east-1,us-east-2
|
91
106
|
```
|
92
107
|
|
93
108
|
```
|
94
|
-
$ AWS_PROFILE=<profile>
|
109
|
+
$ AWS_PROFILE=<profile> aws_recon --services S3,EC2 --regions global,us-east-1,us-east-2
|
95
110
|
```
|
96
111
|
|
97
112
|
#### Errors
|
@@ -118,11 +133,11 @@ For regional services, a thread (up to the thread limit) is spawned for each ser
|
|
118
133
|
Most users will want to limit collection to relevant services and regions. Running without any options will attempt to collect all resources from all 16 regular regions.
|
119
134
|
|
120
135
|
```
|
121
|
-
$
|
136
|
+
$ aws_recon -h
|
122
137
|
|
123
138
|
AWS Recon - AWS Inventory Collector
|
124
139
|
|
125
|
-
Usage:
|
140
|
+
Usage: aws_recon [options]
|
126
141
|
-r, --regions [REGIONS] Regions to scan, separated by comma (default: all)
|
127
142
|
-n, --not-regions [REGIONS] Regions to skip, separated by comma (default: none)
|
128
143
|
-s, --services [SERVICES] Services to scan, separated by comma (default: all)
|
@@ -210,6 +225,21 @@ AWS Recon aims to collect all resources and metadata that are relevant in determ
|
|
210
225
|
|
211
226
|
One of the primary motivations for AWS Recon was to build a tool that is easy to maintain and extend. If you feel like coverage could be improved for a particular service, we would welcome PRs to that effect. Anyone with a moderate familiarity with Ruby will be able to mimic the pattern used by the existing collectors to query a specific service and add the results to the resource collection.
|
212
227
|
|
228
|
+
### Development
|
229
|
+
|
230
|
+
Clone this repository, then install the required gems using `bundle`:
|
231
|
+
|
232
|
+
```
|
233
|
+
$ git clone git@github.com:darkbitio/aws-recon.git
|
234
|
+
$ cd aws-recon
|
235
|
+
$ bundle
|
236
|
+
...
|
237
|
+
Using aws-sdk-core 3.103.0
|
238
|
+
...
|
239
|
+
Bundle complete! 5 Gemfile dependencies, 259 gems now installed.
|
240
|
+
Use `bundle info [gemname]` to see where a bundled gem is installed.
|
241
|
+
```
|
242
|
+
|
213
243
|
### TODO
|
214
244
|
|
215
245
|
- [ ] Optionally suppress AWS API errors instead of re-raising them
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws_recon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Josh Larsen
|
8
|
+
- Darkbit
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2020-08-
|
12
|
+
date: 2020-08-11 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: aws-sdk
|
@@ -70,16 +71,16 @@ dependencies:
|
|
70
71
|
name: rake
|
71
72
|
requirement: !ruby/object:Gem::Requirement
|
72
73
|
requirements:
|
73
|
-
- - "
|
74
|
+
- - ">="
|
74
75
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
76
|
+
version: 12.3.3
|
76
77
|
type: :development
|
77
78
|
prerelease: false
|
78
79
|
version_requirements: !ruby/object:Gem::Requirement
|
79
80
|
requirements:
|
80
|
-
- - "
|
81
|
+
- - ">="
|
81
82
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
83
|
+
version: 12.3.3
|
83
84
|
- !ruby/object:Gem::Dependency
|
84
85
|
name: minitest
|
85
86
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,7 +137,9 @@ dependencies:
|
|
136
137
|
- - "~>"
|
137
138
|
- !ruby/object:Gem::Version
|
138
139
|
version: 0.13.1
|
139
|
-
description:
|
140
|
+
description: AWS Recon is a command line tool to collect resources from an Amazon
|
141
|
+
Web Services (AWS) account. The tool outputs JSON suitable for processing with other
|
142
|
+
tools.
|
140
143
|
email:
|
141
144
|
executables:
|
142
145
|
- aws_recon
|
@@ -241,5 +244,5 @@ requirements: []
|
|
241
244
|
rubygems_version: 3.0.8
|
242
245
|
signing_key:
|
243
246
|
specification_version: 4
|
244
|
-
summary: A multi-threaded AWS inventory collection tool.
|
247
|
+
summary: A multi-threaded AWS inventory collection cli tool.
|
245
248
|
test_files: []
|