aws_recon 0.5.1 → 0.5.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/check-aws-regions.yml +18 -0
- data/aws_recon.gemspec +1 -1
- data/lib/aws_recon/collectors/iam.rb +1 -1
- data/lib/aws_recon/options.rb +2 -2
- data/lib/aws_recon/services.yaml +6 -15
- data/lib/aws_recon/version.rb +1 -1
- data/readme.md +5 -5
- data/utils/aws/check_region_exclusions.rb +112 -0
- data/utils/aws/regions.yaml +43 -0
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 42390b8c429d5b20271c43cca771dd8303ac36625a1854ab03b1979e09efa5eb
|
4
|
+
data.tar.gz: 651d4118fcc2c677f127055caa1c8eef791c60586d24686f2f841ed3635d2398
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 289c6cb8f23196b46b5b2648e285ff161bfbc8dbd3fa634f8056d4d1e300755da798422d576f0b5d2512c1272347c9ea7da9fbd90b6a76345c049ea64e434149
|
7
|
+
data.tar.gz: d0c0239b791945fb2bb7223bc401387c1252c0d806105d85a2f53d0a36cead3e626cdcfb952e3a958213858dcadb55199f999d492b54b7175eccd4af2dbb7faf
|
@@ -0,0 +1,18 @@
|
|
1
|
+
name: check-service-regions
|
2
|
+
|
3
|
+
on:
|
4
|
+
workflow_dispatch:
|
5
|
+
schedule:
|
6
|
+
- cron: '0 10 * * *'
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
region-check:
|
10
|
+
runs-on: ubuntu-20.04
|
11
|
+
steps:
|
12
|
+
- name: Checkout
|
13
|
+
uses: actions/checkout@v2
|
14
|
+
with:
|
15
|
+
fetch-depth: 1
|
16
|
+
- name: Check AWS service regions
|
17
|
+
run: |
|
18
|
+
cd utils/aws ; ruby check_region_exclusions.rb
|
data/aws_recon.gemspec
CHANGED
@@ -8,7 +8,7 @@ Gem::Specification.new do |spec|
|
|
8
8
|
spec.name = 'aws_recon'
|
9
9
|
spec.version = AwsRecon::VERSION
|
10
10
|
spec.authors = ['Josh Larsen', 'Darkbit']
|
11
|
-
spec.required_ruby_version = '>= 2.
|
11
|
+
spec.required_ruby_version = '>= 2.6.0'
|
12
12
|
spec.summary = 'A multi-threaded AWS security-focused inventory collection tool.'
|
13
13
|
spec.description = 'AWS Recon is a command line tool to collect resources from an Amazon Web Services (AWS) account. The tool outputs JSON suitable for processing with other tools.'
|
14
14
|
spec.homepage = 'https://github.com/darkbitio/aws-recon'
|
data/lib/aws_recon/options.rb
CHANGED
@@ -36,8 +36,8 @@ class Parser
|
|
36
36
|
aws_regions = ['global'].concat(Aws::EC2::Client.new.describe_regions.regions.map(&:region_name))
|
37
37
|
end
|
38
38
|
rescue Aws::Errors::ServiceError => e
|
39
|
-
|
40
|
-
exit
|
39
|
+
warn "\nAWS Error: #{e.code}\n\n"
|
40
|
+
exit(1)
|
41
41
|
end
|
42
42
|
|
43
43
|
aws_services = YAML.load(File.read(SERVICES_CONFIG_FILE), symbolize_names: true)
|
data/lib/aws_recon/services.yaml
CHANGED
@@ -37,8 +37,6 @@
|
|
37
37
|
alias: ecs
|
38
38
|
- name: ElasticLoadBalancing
|
39
39
|
alias: elb
|
40
|
-
excluded_regions:
|
41
|
-
- ap-southeast-1
|
42
40
|
- name: ElasticLoadBalancingV2
|
43
41
|
alias: elbv2
|
44
42
|
- name: ElastiCache
|
@@ -58,7 +56,7 @@
|
|
58
56
|
- name: ECR
|
59
57
|
alias: ecr
|
60
58
|
- name: DynamoDB
|
61
|
-
alias:
|
59
|
+
alias: dynamodb
|
62
60
|
- name: KMS
|
63
61
|
alias: kms
|
64
62
|
- name: Kinesis
|
@@ -85,15 +83,15 @@
|
|
85
83
|
- name: Shield
|
86
84
|
global: true
|
87
85
|
alias: shield
|
86
|
+
excluded_regions:
|
87
|
+
- ap-northeast-3
|
88
88
|
- name: CloudFormation
|
89
89
|
alias: cloudformation
|
90
90
|
- name: SES
|
91
91
|
alias: ses
|
92
92
|
excluded_regions:
|
93
|
-
- af-south-1
|
94
93
|
- ap-east-1
|
95
94
|
- ap-northeast-3
|
96
|
-
- eu-south-1
|
97
95
|
- name: CloudWatch
|
98
96
|
alias: cloudwatch
|
99
97
|
- name: CloudWatchLogs
|
@@ -104,11 +102,9 @@
|
|
104
102
|
- af-south-1
|
105
103
|
- ap-northeast-3
|
106
104
|
- name: SecretsManager
|
107
|
-
alias:
|
105
|
+
alias: secretsmanager
|
108
106
|
- name: SecurityHub
|
109
|
-
alias:
|
110
|
-
excluded_regions:
|
111
|
-
- ap-northeast-3
|
107
|
+
alias: securityhub
|
112
108
|
- name: Support
|
113
109
|
global: true
|
114
110
|
alias: support
|
@@ -116,16 +112,12 @@
|
|
116
112
|
alias: ssm
|
117
113
|
- name: GuardDuty
|
118
114
|
alias: guardduty
|
119
|
-
excluded_regions:
|
120
|
-
- ap-northeast-3
|
121
115
|
- name: Athena
|
122
116
|
alias: athena
|
123
117
|
excluded_regions:
|
124
118
|
- ap-northeast-3
|
125
119
|
- name: EFS
|
126
120
|
alias: efs
|
127
|
-
excluded_regions:
|
128
|
-
- ap-northeast-3
|
129
121
|
- name: Firehose
|
130
122
|
alias: firehose
|
131
123
|
- name: Lightsail
|
@@ -145,7 +137,6 @@
|
|
145
137
|
- af-south-1
|
146
138
|
- ap-east-1
|
147
139
|
- ap-northeast-3
|
148
|
-
- ap-south-1
|
149
140
|
- eu-north-1
|
150
141
|
- eu-south-1
|
151
142
|
- eu-west-3
|
@@ -164,7 +155,7 @@
|
|
164
155
|
- ap-northeast-3
|
165
156
|
- eu-south-1
|
166
157
|
- name: DirectConnect
|
167
|
-
alias:
|
158
|
+
alias: directconnect
|
168
159
|
- name: DirectoryService
|
169
160
|
alias: ds
|
170
161
|
excluded_regions:
|
data/lib/aws_recon/version.rb
CHANGED
data/readme.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/darkbitio/aws-recon/smoke-test/main)](https://github.com/darkbitio/aws-recon/actions?query=branch%3Amain)
|
2
2
|
[![Gem Version](https://badge.fury.io/rb/aws_recon.svg)](https://rubygems.org/gems/aws_recon)
|
3
|
-
|
3
|
+
[![AWS Service Regions](https://github.com/darkbitio/aws-recon/actions/workflows/check-aws-regions.yml/badge.svg?branch=main&event=schedule)](https://github.com/darkbitio/aws-recon/actions/workflows/check-aws-regions.yml)
|
4
4
|
# AWS Recon
|
5
5
|
|
6
6
|
A multi-threaded AWS security-focused inventory collection tool written in Ruby.
|
@@ -54,13 +54,13 @@ To run locally, first install the gem:
|
|
54
54
|
|
55
55
|
```
|
56
56
|
$ gem install aws_recon
|
57
|
-
Fetching aws_recon-0.
|
57
|
+
Fetching aws_recon-0.5.2.gem
|
58
58
|
Fetching aws-sdk-3.0.1.gem
|
59
59
|
Fetching parallel-1.20.1.gem
|
60
60
|
...
|
61
61
|
Successfully installed aws-sdk-3.0.1
|
62
62
|
Successfully installed parallel-1.20.1
|
63
|
-
Successfully installed aws_recon-0.
|
63
|
+
Successfully installed aws_recon-0.5.2
|
64
64
|
```
|
65
65
|
|
66
66
|
Or add it to your Gemfile using `bundle`:
|
@@ -72,7 +72,7 @@ Resolving dependencies...
|
|
72
72
|
...
|
73
73
|
Using aws-sdk 3.0.1
|
74
74
|
Using parallel-1.20.1
|
75
|
-
Using aws_recon 0.
|
75
|
+
Using aws_recon 0.5.2
|
76
76
|
```
|
77
77
|
|
78
78
|
## Usage
|
@@ -249,7 +249,7 @@ Most users will want to limit collection to relevant services and regions. Runni
|
|
249
249
|
```
|
250
250
|
$ aws_recon -h
|
251
251
|
|
252
|
-
AWS Recon - AWS Inventory Collector (0.
|
252
|
+
AWS Recon - AWS Inventory Collector (0.5.2)
|
253
253
|
|
254
254
|
Usage: aws_recon [options]
|
255
255
|
-r, --regions [REGIONS] Regions to scan, separated by comma (default: all)
|
@@ -0,0 +1,112 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Check regional service availability against services.yaml exclusions.
|
5
|
+
#
|
6
|
+
# AWS updates the regional service table daily. By checking regional service
|
7
|
+
# coverage, we can identify regions that should be excluded from AWS Recon
|
8
|
+
# checks. We exclude non-supported regions because service APIs handle non-
|
9
|
+
# availability differently. Some will respond with an error that can be handled
|
10
|
+
# by the errors defined in the AWS Ruby SDK client. Others will fail at the
|
11
|
+
# network level (i.e. there is no API endpoint even available). We could handle
|
12
|
+
# those errors and silently fail, but we choose not to so we can identify cases
|
13
|
+
# where there is a lack of service availability in a particular region.
|
14
|
+
#
|
15
|
+
require 'net/http'
|
16
|
+
require 'json'
|
17
|
+
require 'yaml'
|
18
|
+
|
19
|
+
TS = Time.now.to_i
|
20
|
+
# AWS Regional services table
|
21
|
+
URL = "https://api.regional-table.region-services.aws.a2z.com/index.json?timestamp=#{TS}000"
|
22
|
+
|
23
|
+
service_to_query = ARGV[0]
|
24
|
+
region_exclusion_mistmatch = nil
|
25
|
+
|
26
|
+
#
|
27
|
+
# load current AWS Recon regions
|
28
|
+
#
|
29
|
+
recon_services = YAML.safe_load(File.read('../../lib/aws_recon/services.yaml'))
|
30
|
+
abort('Errors loading AWS Recon services') unless recon_services.is_a?(Array)
|
31
|
+
|
32
|
+
#
|
33
|
+
# load current AWS regions (non-gov, non-cn)
|
34
|
+
#
|
35
|
+
regions = YAML.safe_load(File.read('regions.yaml'))
|
36
|
+
abort('Errors loading regions') unless regions['Regions']
|
37
|
+
|
38
|
+
all_regions = regions['Regions'].map { |r| r['RegionName'] }
|
39
|
+
|
40
|
+
#
|
41
|
+
# get service/price list from AWS
|
42
|
+
#
|
43
|
+
uri = URI(URL)
|
44
|
+
res = Net::HTTP.get_response(uri)
|
45
|
+
abort('Error loading AWS services from API') unless res.code == '200'
|
46
|
+
|
47
|
+
map = {}
|
48
|
+
|
49
|
+
#
|
50
|
+
# load service region availability
|
51
|
+
#
|
52
|
+
data = res.body
|
53
|
+
json = JSON.parse(data)
|
54
|
+
|
55
|
+
#
|
56
|
+
# query regions for a single service
|
57
|
+
#
|
58
|
+
if service_to_query
|
59
|
+
single_service_regions = []
|
60
|
+
|
61
|
+
json['prices'].each do |p|
|
62
|
+
single_service_regions << p['id'].split(':').last
|
63
|
+
end
|
64
|
+
|
65
|
+
single_service_regions.uniq.sort.each { |r| puts r }
|
66
|
+
|
67
|
+
exit 0
|
68
|
+
end
|
69
|
+
|
70
|
+
# iterate through AWS provided services & regions
|
71
|
+
json['prices'].each do |p|
|
72
|
+
at = p['attributes']
|
73
|
+
service_name = at['aws:serviceName']
|
74
|
+
service_id, service_region = p['id'].split(':')
|
75
|
+
|
76
|
+
# skip this service unless AWS Recon already has exclusions
|
77
|
+
next unless recon_services.filter { |s| s['alias'] == service_id }&.length&.positive?
|
78
|
+
|
79
|
+
if map.key?(service_name)
|
80
|
+
map[service_name]['regions'] << service_region
|
81
|
+
else
|
82
|
+
map[service_name] = {
|
83
|
+
'id' => service_id,
|
84
|
+
'regions' => [service_region]
|
85
|
+
}
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# iterate through the services AWS Recon knows about
|
90
|
+
map.sort.each do |k, v|
|
91
|
+
service_excluded_regions = all_regions.reject { |r| v['regions'].include?(r) }
|
92
|
+
|
93
|
+
aws_recon_service = recon_services.filter { |s| s['alias'] == v['id'] }&.first
|
94
|
+
aws_recon_service_excluded_regions = aws_recon_service['excluded_regions'] || []
|
95
|
+
|
96
|
+
# move on if AWS Recon region exclusions match AWS service region exclusions
|
97
|
+
next unless service_excluded_regions.sort != aws_recon_service_excluded_regions.sort
|
98
|
+
|
99
|
+
region_exclusion_mistmatch = true
|
100
|
+
|
101
|
+
puts "#{k} (#{v['id']})"
|
102
|
+
|
103
|
+
# determine the direction of the exclusion mismatch
|
104
|
+
if (service_excluded_regions - aws_recon_service_excluded_regions).length.positive?
|
105
|
+
puts " + missing region exclusion: #{(service_excluded_regions - aws_recon_service_excluded_regions).join(', ')}"
|
106
|
+
else
|
107
|
+
puts " - unnecessary region exclusion: #{(aws_recon_service_excluded_regions - service_excluded_regions).join(', ')}"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
# exit code 1 if we have any mismatches
|
112
|
+
exit 1 if region_exclusion_mistmatch
|
@@ -0,0 +1,43 @@
|
|
1
|
+
Regions:
|
2
|
+
- Endpoint: ec2.af-south-1.amazonaws.com
|
3
|
+
RegionName: af-south-1
|
4
|
+
- Endpoint: ec2.eu-north-1.amazonaws.com
|
5
|
+
RegionName: eu-north-1
|
6
|
+
- Endpoint: ec2.ap-south-1.amazonaws.com
|
7
|
+
RegionName: ap-south-1
|
8
|
+
- Endpoint: ec2.eu-west-3.amazonaws.com
|
9
|
+
RegionName: eu-west-3
|
10
|
+
- Endpoint: ec2.eu-west-2.amazonaws.com
|
11
|
+
RegionName: eu-west-2
|
12
|
+
- Endpoint: ec2.eu-south-1.amazonaws.com
|
13
|
+
RegionName: eu-south-1
|
14
|
+
- Endpoint: ec2.eu-west-1.amazonaws.com
|
15
|
+
RegionName: eu-west-1
|
16
|
+
- Endpoint: ec2.ap-northeast-3.amazonaws.com
|
17
|
+
RegionName: ap-northeast-3
|
18
|
+
- Endpoint: ec2.ap-northeast-2.amazonaws.com
|
19
|
+
RegionName: ap-northeast-2
|
20
|
+
- Endpoint: ec2.me-south-1.amazonaws.com
|
21
|
+
RegionName: me-south-1
|
22
|
+
- Endpoint: ec2.ap-northeast-1.amazonaws.com
|
23
|
+
RegionName: ap-northeast-1
|
24
|
+
- Endpoint: ec2.sa-east-1.amazonaws.com
|
25
|
+
RegionName: sa-east-1
|
26
|
+
- Endpoint: ec2.ca-central-1.amazonaws.com
|
27
|
+
RegionName: ca-central-1
|
28
|
+
- Endpoint: ec2.ap-east-1.amazonaws.com
|
29
|
+
RegionName: ap-east-1
|
30
|
+
- Endpoint: ec2.ap-southeast-1.amazonaws.com
|
31
|
+
RegionName: ap-southeast-1
|
32
|
+
- Endpoint: ec2.ap-southeast-2.amazonaws.com
|
33
|
+
RegionName: ap-southeast-2
|
34
|
+
- Endpoint: ec2.eu-central-1.amazonaws.com
|
35
|
+
RegionName: eu-central-1
|
36
|
+
- Endpoint: ec2.us-east-1.amazonaws.com
|
37
|
+
RegionName: us-east-1
|
38
|
+
- Endpoint: ec2.us-east-2.amazonaws.com
|
39
|
+
RegionName: us-east-2
|
40
|
+
- Endpoint: ec2.us-west-1.amazonaws.com
|
41
|
+
RegionName: us-west-1
|
42
|
+
- Endpoint: ec2.us-west-2.amazonaws.com
|
43
|
+
RegionName: us-west-2
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws_recon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Josh Larsen
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-04-
|
12
|
+
date: 2021-04-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: aws-sdk
|
@@ -163,6 +163,7 @@ extensions: []
|
|
163
163
|
extra_rdoc_files: []
|
164
164
|
files:
|
165
165
|
- ".github/stale.yml"
|
166
|
+
- ".github/workflows/check-aws-regions.yml"
|
166
167
|
- ".github/workflows/docker-build.yml"
|
167
168
|
- ".github/workflows/smoke-test.yml"
|
168
169
|
- ".gitignore"
|
@@ -245,6 +246,8 @@ files:
|
|
245
246
|
- lib/aws_recon/services.yaml
|
246
247
|
- lib/aws_recon/version.rb
|
247
248
|
- readme.md
|
249
|
+
- utils/aws/check_region_exclusions.rb
|
250
|
+
- utils/aws/regions.yaml
|
248
251
|
- utils/cloudformation/aws-recon-cfn-template.yml
|
249
252
|
- utils/terraform/cloudwatch.tf
|
250
253
|
- utils/terraform/ecs.tf
|
@@ -267,7 +270,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
267
270
|
requirements:
|
268
271
|
- - ">="
|
269
272
|
- !ruby/object:Gem::Version
|
270
|
-
version: 2.
|
273
|
+
version: 2.6.0
|
271
274
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
272
275
|
requirements:
|
273
276
|
- - ">="
|