aquatone 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/CHANGELOG.md +18 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +168 -0
- data/Rakefile +10 -0
- data/aquatone.gemspec +29 -0
- data/aquatone.js +164 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/aquatone-discover +129 -0
- data/exe/aquatone-gather +55 -0
- data/exe/aquatone-scan +76 -0
- data/lib/aquatone.rb +43 -0
- data/lib/aquatone/assessment.rb +40 -0
- data/lib/aquatone/browser.rb +18 -0
- data/lib/aquatone/browser/drivers/nightmare.rb +52 -0
- data/lib/aquatone/collector.rb +106 -0
- data/lib/aquatone/collectors/dictionary.rb +20 -0
- data/lib/aquatone/collectors/dnsdb.rb +45 -0
- data/lib/aquatone/collectors/gtr.rb +58 -0
- data/lib/aquatone/collectors/hackertarget.rb +24 -0
- data/lib/aquatone/collectors/netcraft.rb +48 -0
- data/lib/aquatone/collectors/shodan.rb +45 -0
- data/lib/aquatone/collectors/threatcrowd.rb +25 -0
- data/lib/aquatone/collectors/virustotal.rb +24 -0
- data/lib/aquatone/command.rb +152 -0
- data/lib/aquatone/commands/discover.rb +187 -0
- data/lib/aquatone/commands/gather.rb +167 -0
- data/lib/aquatone/commands/scan.rb +108 -0
- data/lib/aquatone/domain.rb +33 -0
- data/lib/aquatone/http_client.rb +5 -0
- data/lib/aquatone/key_store.rb +72 -0
- data/lib/aquatone/port_lists.rb +36 -0
- data/lib/aquatone/report.rb +88 -0
- data/lib/aquatone/resolver.rb +47 -0
- data/lib/aquatone/thread_pool.rb +31 -0
- data/lib/aquatone/url_maker.rb +27 -0
- data/lib/aquatone/validation.rb +22 -0
- data/lib/aquatone/version.rb +3 -0
- data/subdomains.lst +8214 -0
- data/templates/default.html.erb +225 -0
- metadata +159 -0
@@ -0,0 +1,187 @@
|
|
1
|
+
module Aquatone
|
2
|
+
module Commands
|
3
|
+
class Discover < Aquatone::Command
|
4
|
+
def execute!
|
5
|
+
if !options[:domain]
|
6
|
+
output("Please specify a domain to assess\n")
|
7
|
+
exit 1
|
8
|
+
end
|
9
|
+
|
10
|
+
@domain = Aquatone::Domain.new(options[:domain])
|
11
|
+
@assessment = Aquatone::Assessment.new(options[:domain])
|
12
|
+
@hosts = [options[:domain]]
|
13
|
+
@host_dictionary = {}
|
14
|
+
|
15
|
+
banner("Discover")
|
16
|
+
setup_resolver
|
17
|
+
identify_wildcard_ips
|
18
|
+
run_collectors
|
19
|
+
resolve_hosts
|
20
|
+
output_summary
|
21
|
+
write_to_hosts_file
|
22
|
+
rescue Aquatone::Domain::UnresolvableDomain => e
|
23
|
+
output(red("Error: #{e.message}\n"))
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def setup_resolver
|
29
|
+
if options[:nameservers]
|
30
|
+
nameservers = options[:nameservers]
|
31
|
+
else
|
32
|
+
output("Identifying nameservers for #{@domain.name}... ")
|
33
|
+
nameservers = @domain.nameservers
|
34
|
+
output("Done\n")
|
35
|
+
if nameservers.empty?
|
36
|
+
output(yellow("#{@domain.name} has no nameservers. Using fallback nameservers.\n\n"))
|
37
|
+
nameservers = []
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
if !nameservers.empty?
|
42
|
+
output("Using nameservers:\n\n")
|
43
|
+
nameservers.each do |ns|
|
44
|
+
output(" - #{ns}\n")
|
45
|
+
end
|
46
|
+
output("\n")
|
47
|
+
end
|
48
|
+
@resolver = Aquatone::Resolver.new(
|
49
|
+
:nameservers => nameservers,
|
50
|
+
:fallback_nameservers => options[:fallback_nameservers]
|
51
|
+
)
|
52
|
+
end
|
53
|
+
|
54
|
+
def identify_wildcard_ips
|
55
|
+
output("Checking for wildcard DNS... ")
|
56
|
+
@wildcard_ips = []
|
57
|
+
wildcard_domain = "#{random_string}.#{@domain.name}"
|
58
|
+
if @resolver.resolve(wildcard_domain).nil?
|
59
|
+
output("Done\n")
|
60
|
+
return
|
61
|
+
end
|
62
|
+
output(yellow("Wildcard detected!\n"))
|
63
|
+
output("Identifying wildcard IPs... ")
|
64
|
+
20.times do
|
65
|
+
wildcard_domain = "#{random_string}.#{@domain.name}"
|
66
|
+
if wildcard_ip = @resolver.resolve(wildcard_domain)
|
67
|
+
@wildcard_ips << wildcard_ip unless @wildcard_ips.include?(wildcard_ip)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
output("Done\n")
|
71
|
+
output("Filtering out hosts resolving to wildcard IPs\n")
|
72
|
+
end
|
73
|
+
|
74
|
+
def run_collectors
|
75
|
+
output("\n")
|
76
|
+
Aquatone::Collector.descendants.each do |collector|
|
77
|
+
next if skip_collector?(collector)
|
78
|
+
output("Running collector: #{bold(collector.meta[:name])}... ")
|
79
|
+
begin
|
80
|
+
collector_instance = collector.new(@domain)
|
81
|
+
hosts = collector_instance.execute!
|
82
|
+
output("Done (#{hosts.count} #{hosts.count == 1 ? 'host' : 'hosts'})\n")
|
83
|
+
@hosts += hosts
|
84
|
+
rescue Aquatone::Collector::MissingKeyRequirement => e
|
85
|
+
output(yellow("Skipped\n"))
|
86
|
+
output(yellow(" -> #{e.message}\n"))
|
87
|
+
rescue => e
|
88
|
+
output(red("Error\n"))
|
89
|
+
output(red(" -> #{e.message}\n"))
|
90
|
+
end
|
91
|
+
end
|
92
|
+
@hosts = @hosts.sort.uniq
|
93
|
+
end
|
94
|
+
|
95
|
+
def resolve_hosts
|
96
|
+
output("\nResolving #{bold(@hosts.count)} unique hosts...\n")
|
97
|
+
task_count = 0
|
98
|
+
@start_time = Time.now.to_i
|
99
|
+
@hosts.each do |host|
|
100
|
+
if asked_for_progress?
|
101
|
+
output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
|
102
|
+
"#{task_count} out of #{@hosts.count} hosts checked (#{@host_dictionary.keys.count} discovered); " \
|
103
|
+
"#{(task_count.to_f / @hosts.count.to_f * 100.00).round(1)}% done\n")
|
104
|
+
end
|
105
|
+
if ip = @resolver.resolve(host)
|
106
|
+
next if wildcard_ip?(ip)
|
107
|
+
next if (options[:ignore_private] && private_ip?(ip))
|
108
|
+
@host_dictionary[host] = ip
|
109
|
+
output("#{ip.ljust(15)} #{bold(host)}\n")
|
110
|
+
end
|
111
|
+
jitter_sleep
|
112
|
+
task_count += 1
|
113
|
+
end
|
114
|
+
output("\n", true)
|
115
|
+
end
|
116
|
+
|
117
|
+
def output_summary
|
118
|
+
subnets = find_subnets
|
119
|
+
if !subnets.keys.count.zero?
|
120
|
+
output("Found subnets:\n\n")
|
121
|
+
subnets.each_pair do |subnet, count|
|
122
|
+
next if count == 1
|
123
|
+
subnet = "#{subnet}.0-255"
|
124
|
+
output(" - #{subnet.ljust(17)} : #{count} hosts\n")
|
125
|
+
end
|
126
|
+
end
|
127
|
+
output("\n")
|
128
|
+
end
|
129
|
+
|
130
|
+
def find_subnets
|
131
|
+
subnets = {}
|
132
|
+
@host_dictionary.values.each do |ip|
|
133
|
+
subnet = ip.split(".")[0..2].join(".")
|
134
|
+
if subnets.key?(subnet)
|
135
|
+
subnets[subnet] += 1
|
136
|
+
else
|
137
|
+
subnets[subnet] = 1
|
138
|
+
end
|
139
|
+
end
|
140
|
+
Hash[subnets.sort_by{|k, v| v}.reverse]
|
141
|
+
end
|
142
|
+
|
143
|
+
def write_to_hosts_file
|
144
|
+
@hosts_file_contents = ""
|
145
|
+
@host_dictionary.each_pair do |host, ip|
|
146
|
+
@hosts_file_contents += "#{host},#{ip}\n"
|
147
|
+
end
|
148
|
+
@assessment.write_file("hosts.txt", @hosts_file_contents)
|
149
|
+
@assessment.write_file("hosts.json", @host_dictionary.to_json)
|
150
|
+
output("Wrote #{bold(@host_dictionary.keys.count)} hosts to:\n\n")
|
151
|
+
output(" - #{bold('file://' + File.join(@assessment.path, 'hosts.txt'))}\n")
|
152
|
+
output(" - #{bold('file://' + File.join(@assessment.path, 'hosts.json'))}\n")
|
153
|
+
end
|
154
|
+
|
155
|
+
def random_string
|
156
|
+
%w(a b c d e f g h i j k l m n o p q r s t u v w x y z
|
157
|
+
0 1 2 3 4 5 6 7 8 9).shuffle.take(10).join
|
158
|
+
end
|
159
|
+
|
160
|
+
def wildcard_ip?(ip)
|
161
|
+
@wildcard_ips.include?(ip)
|
162
|
+
end
|
163
|
+
|
164
|
+
def private_ip?(ip)
|
165
|
+
ip =~ /(\A127\.)|(\A10\.)|(\A172\.1[6-9]\.)|(\A172\.2[0-9]\.)|(\A172\.3[0-1]\.)|(\A192\.168\.)/
|
166
|
+
end
|
167
|
+
|
168
|
+
def skip_collector?(collector)
|
169
|
+
if options[:only_collectors]
|
170
|
+
if options[:only_collectors].include?(collector.sluggified_name)
|
171
|
+
false
|
172
|
+
else
|
173
|
+
true
|
174
|
+
end
|
175
|
+
elsif options[:disable_collectors]
|
176
|
+
if options[:disable_collectors].include?(collector.sluggified_name)
|
177
|
+
true
|
178
|
+
else
|
179
|
+
false
|
180
|
+
end
|
181
|
+
else
|
182
|
+
false
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
module Aquatone
|
2
|
+
module Commands
|
3
|
+
class Gather < Aquatone::Command
|
4
|
+
def execute!
|
5
|
+
if !options[:domain]
|
6
|
+
output("Please specify a domain to assess\n")
|
7
|
+
exit 1
|
8
|
+
end
|
9
|
+
|
10
|
+
@assessment = Aquatone::Assessment.new(options[:domain])
|
11
|
+
|
12
|
+
banner("Gather")
|
13
|
+
check_prerequisites
|
14
|
+
prepare_tasks
|
15
|
+
make_directories
|
16
|
+
process_pages
|
17
|
+
generate_report
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def check_prerequisites
|
23
|
+
if !has_executable?("node")
|
24
|
+
output(red("node executable not found!\n\n"))
|
25
|
+
output("Please make sure Node.js is installed on your system.\n")
|
26
|
+
exit 1
|
27
|
+
end
|
28
|
+
|
29
|
+
if !has_executable?("npm")
|
30
|
+
output(red("npm executable not found!\n\n"))
|
31
|
+
output("Please make sure NPM package manager is installed on your system.\n")
|
32
|
+
exit 1
|
33
|
+
end
|
34
|
+
|
35
|
+
if !Dir.exists?(File.join(Aquatone::AQUATONE_ROOT, "node_modules"))
|
36
|
+
output("Installing Nightmare.js package, please wait...")
|
37
|
+
Dir.chdir(Aquatone::AQUATONE_ROOT) do
|
38
|
+
if system("npm install nightmare >/dev/null 2>&1")
|
39
|
+
output(" Done\n\n")
|
40
|
+
else
|
41
|
+
output(red(" Failed\n"))
|
42
|
+
exit 1
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def prepare_tasks
|
49
|
+
if !@assessment.has_file?("hosts.json")
|
50
|
+
output(red("#{@assessment.path} does not contain hosts.json file\n\n"))
|
51
|
+
output("Did you run aquatone-discover first?\n")
|
52
|
+
exit 1
|
53
|
+
end
|
54
|
+
if !@assessment.has_file?("open_ports.txt")
|
55
|
+
output(red("#{@assessment.path} does not contain open_ports.txt file\n\n"))
|
56
|
+
output("Did you run aquatone-scan first?\n")
|
57
|
+
exit 1
|
58
|
+
end
|
59
|
+
@tasks = []
|
60
|
+
@hosts = JSON.parse(@assessment.read_file("hosts.json"))
|
61
|
+
@open_ports = parse_open_ports_file
|
62
|
+
@hosts.each_pair do |domain, host|
|
63
|
+
next unless @open_ports.key?(host)
|
64
|
+
@open_ports[host].each do |port|
|
65
|
+
@tasks << [host, port, domain]
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def process_pages
|
71
|
+
pool = thread_pool
|
72
|
+
@task_count = 0
|
73
|
+
@successful = 0
|
74
|
+
@failed = 0
|
75
|
+
@start_time = Time.now.to_i
|
76
|
+
@visits = []
|
77
|
+
output("Processing #{bold(@tasks.count)} pages...\n")
|
78
|
+
@tasks.shuffle.each do |task|
|
79
|
+
host, port, domain = task
|
80
|
+
pool.schedule do
|
81
|
+
output_progress if asked_for_progress?
|
82
|
+
visit = visit_page(host, port, domain)
|
83
|
+
if visit['success']
|
84
|
+
output("#{green('Processed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['status']}\n")
|
85
|
+
@successful += 1
|
86
|
+
else
|
87
|
+
output(" #{red('Failed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['error']} #{visit['details']}\n")
|
88
|
+
@failed += 1
|
89
|
+
end
|
90
|
+
jitter_sleep
|
91
|
+
@task_count += 1
|
92
|
+
end
|
93
|
+
end
|
94
|
+
pool.shutdown
|
95
|
+
output("\nFinished processing pages:\n\n")
|
96
|
+
output(" - Successful : #{bold(green(@successful))}\n")
|
97
|
+
output(" - Failed : #{bold(red(@failed))}\n\n")
|
98
|
+
end
|
99
|
+
|
100
|
+
def generate_report
|
101
|
+
output("Generating report...")
|
102
|
+
report = Aquatone::Report.new(options[:domain], @visits)
|
103
|
+
report.generate(File.join(@assessment.path, "report"))
|
104
|
+
output("done\n")
|
105
|
+
report_pages = Dir[File.join(@assessment.path, "report", "report_page_*.html")]
|
106
|
+
output("Report pages generated:\n\n")
|
107
|
+
sort_report_pages(report_pages).each do |report_page|
|
108
|
+
output(" - file://#{report_page}\n")
|
109
|
+
end
|
110
|
+
output("\n")
|
111
|
+
end
|
112
|
+
|
113
|
+
def parse_open_ports_file
|
114
|
+
contents = @assessment.read_file("open_ports.txt")
|
115
|
+
result = {}
|
116
|
+
lines = contents.split("\n").map(&:strip)
|
117
|
+
lines.each do |line|
|
118
|
+
values = line.split(",").map(&:strip)
|
119
|
+
result[values[0]] = values[1..-1].map(&:to_i)
|
120
|
+
end
|
121
|
+
result
|
122
|
+
end
|
123
|
+
|
124
|
+
def make_directories
|
125
|
+
@assessment.make_directory("headers")
|
126
|
+
@assessment.make_directory("html")
|
127
|
+
@assessment.make_directory("report")
|
128
|
+
@assessment.make_directory("screenshots")
|
129
|
+
end
|
130
|
+
|
131
|
+
def make_file_basename(host, port, domain)
|
132
|
+
"#{domain}__#{host}__#{port}".downcase.gsub(".", "_")
|
133
|
+
end
|
134
|
+
|
135
|
+
def output_progress
|
136
|
+
output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
|
137
|
+
"#{@task_count} out of #{@tasks.count} pages processed (#{@successful} successful, #{@failed} failed); " \
|
138
|
+
"#{(@task_count.to_f / @tasks.count.to_f * 100.00).round(1)}% done\n")
|
139
|
+
end
|
140
|
+
|
141
|
+
def sort_report_pages(pages)
|
142
|
+
pages.sort_by { |f| File.basename(f).split("_").last.split(".").first.to_i }
|
143
|
+
end
|
144
|
+
|
145
|
+
def visit_page(host, port, domain)
|
146
|
+
file_basename = make_file_basename(host, port, domain)
|
147
|
+
url = Aquatone::UrlMaker.make(host, port)
|
148
|
+
html_destination = File.join(@assessment.path, "html", "#{file_basename}.html")
|
149
|
+
headers_destination = File.join(@assessment.path, "headers", "#{file_basename}.txt")
|
150
|
+
screenshot_destination = File.join(@assessment.path, "screenshots", "#{file_basename}.png")
|
151
|
+
visit = Aquatone::Browser.visit(url, domain, html_destination, headers_destination, screenshot_destination, :timeout => options[:timeout])
|
152
|
+
if visit['success']
|
153
|
+
@visits.push({
|
154
|
+
:host => host,
|
155
|
+
:port => port,
|
156
|
+
:domain => domain,
|
157
|
+
:url => url,
|
158
|
+
:file_basename => file_basename,
|
159
|
+
:headers => visit['headers'],
|
160
|
+
:status => visit['status']
|
161
|
+
})
|
162
|
+
end
|
163
|
+
visit
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
module Aquatone
|
2
|
+
module Commands
|
3
|
+
class Scan < Aquatone::Command
|
4
|
+
def execute!
|
5
|
+
if !options[:domain]
|
6
|
+
output("Please specify a domain to assess\n")
|
7
|
+
exit 1
|
8
|
+
end
|
9
|
+
|
10
|
+
@assessment = Aquatone::Assessment.new(options[:domain])
|
11
|
+
@tasks = []
|
12
|
+
@host_dictionary = {}
|
13
|
+
@results = {}
|
14
|
+
@urls = []
|
15
|
+
|
16
|
+
banner("Scan")
|
17
|
+
prepare_host_dictionary
|
18
|
+
scan_ports
|
19
|
+
write_open_ports_file
|
20
|
+
write_urls_file
|
21
|
+
end
|
22
|
+
|
23
|
+
def prepare_host_dictionary
|
24
|
+
if !@assessment.has_file?("hosts.json")
|
25
|
+
output(red("#{@assessment.path} does not contain hosts.json file\n\n"))
|
26
|
+
output("Did you run aquatone-discover first?\n")
|
27
|
+
exit 1
|
28
|
+
end
|
29
|
+
hosts = JSON.parse(@assessment.read_file("hosts.json"))
|
30
|
+
output("Loaded #{bold(hosts.count)} hosts from #{bold(File.join(@assessment.path, 'hosts.json'))}\n\n")
|
31
|
+
hosts.each_pair do |domain, ip|
|
32
|
+
if !@host_dictionary.key?(ip)
|
33
|
+
@host_dictionary[ip] = [domain]
|
34
|
+
options[:ports].each do |port|
|
35
|
+
@tasks << [ip, port]
|
36
|
+
end
|
37
|
+
else
|
38
|
+
@host_dictionary[ip] << domain
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def scan_ports
|
44
|
+
pool = thread_pool
|
45
|
+
@task_count = 1
|
46
|
+
@ports_open = 0
|
47
|
+
@start_time = Time.now.to_i
|
48
|
+
output("Probing #{bold(@tasks.count)} ports...\n")
|
49
|
+
@tasks.shuffle.each do |task|
|
50
|
+
host, port = task
|
51
|
+
pool.schedule do
|
52
|
+
output_progress if asked_for_progress?
|
53
|
+
if port_open?(host, port)
|
54
|
+
output_open_port(host, port)
|
55
|
+
@ports_open += 1
|
56
|
+
@results[host] ||= []
|
57
|
+
@results[host] << port
|
58
|
+
@host_dictionary[host].each do |hostname|
|
59
|
+
@urls << Aquatone::UrlMaker.make(hostname, port)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
jitter_sleep
|
63
|
+
@task_count += 1
|
64
|
+
end
|
65
|
+
end
|
66
|
+
pool.shutdown
|
67
|
+
end
|
68
|
+
|
69
|
+
def write_open_ports_file
|
70
|
+
contents = []
|
71
|
+
@results.each_pair do |host, ports|
|
72
|
+
contents << "#{host},#{ports.join(',')}"
|
73
|
+
end
|
74
|
+
@assessment.write_file("open_ports.txt", contents.sort.join("\n"))
|
75
|
+
output("\nWrote open ports to #{bold('file://' + File.join(@assessment.path, 'open_ports.txt'))}\n")
|
76
|
+
end
|
77
|
+
|
78
|
+
def write_urls_file
|
79
|
+
@assessment.write_file("urls.txt", @urls.uniq.sort.join("\n"))
|
80
|
+
output("Wrote URLs to #{bold('file://' + File.join(@assessment.path, 'urls.txt'))}\n")
|
81
|
+
end
|
82
|
+
|
83
|
+
def port_open?(host, port)
|
84
|
+
Timeout::timeout(options[:timeout]) do
|
85
|
+
TCPSocket.new(host, port).close
|
86
|
+
true
|
87
|
+
end
|
88
|
+
rescue Timeout::Error, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError
|
89
|
+
false
|
90
|
+
end
|
91
|
+
|
92
|
+
def output_progress
|
93
|
+
output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
|
94
|
+
"#{@task_count} out of #{@tasks.count} ports checked (#{@ports_open} open); " \
|
95
|
+
"#{(@task_count.to_f / @tasks.count.to_f * 100.00).round(1)}% done\n")
|
96
|
+
end
|
97
|
+
|
98
|
+
def output_open_port(host, port)
|
99
|
+
if (@host_dictionary[host].count > 3)
|
100
|
+
domains = @host_dictionary[host].shuffle.take(3).join(", ") + " and #{@host_dictionary[host].count - 3} more"
|
101
|
+
else
|
102
|
+
domains = @host_dictionary[host].take(3).join(", ")
|
103
|
+
end
|
104
|
+
output("#{green((port.to_s + '/tcp').ljust(9))} #{host.ljust(15)} #{domains}\n")
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|