aquatone 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/CHANGELOG.md +18 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +168 -0
  7. data/Rakefile +10 -0
  8. data/aquatone.gemspec +29 -0
  9. data/aquatone.js +164 -0
  10. data/bin/console +14 -0
  11. data/bin/setup +8 -0
  12. data/exe/aquatone-discover +129 -0
  13. data/exe/aquatone-gather +55 -0
  14. data/exe/aquatone-scan +76 -0
  15. data/lib/aquatone.rb +43 -0
  16. data/lib/aquatone/assessment.rb +40 -0
  17. data/lib/aquatone/browser.rb +18 -0
  18. data/lib/aquatone/browser/drivers/nightmare.rb +52 -0
  19. data/lib/aquatone/collector.rb +106 -0
  20. data/lib/aquatone/collectors/dictionary.rb +20 -0
  21. data/lib/aquatone/collectors/dnsdb.rb +45 -0
  22. data/lib/aquatone/collectors/gtr.rb +58 -0
  23. data/lib/aquatone/collectors/hackertarget.rb +24 -0
  24. data/lib/aquatone/collectors/netcraft.rb +48 -0
  25. data/lib/aquatone/collectors/shodan.rb +45 -0
  26. data/lib/aquatone/collectors/threatcrowd.rb +25 -0
  27. data/lib/aquatone/collectors/virustotal.rb +24 -0
  28. data/lib/aquatone/command.rb +152 -0
  29. data/lib/aquatone/commands/discover.rb +187 -0
  30. data/lib/aquatone/commands/gather.rb +167 -0
  31. data/lib/aquatone/commands/scan.rb +108 -0
  32. data/lib/aquatone/domain.rb +33 -0
  33. data/lib/aquatone/http_client.rb +5 -0
  34. data/lib/aquatone/key_store.rb +72 -0
  35. data/lib/aquatone/port_lists.rb +36 -0
  36. data/lib/aquatone/report.rb +88 -0
  37. data/lib/aquatone/resolver.rb +47 -0
  38. data/lib/aquatone/thread_pool.rb +31 -0
  39. data/lib/aquatone/url_maker.rb +27 -0
  40. data/lib/aquatone/validation.rb +22 -0
  41. data/lib/aquatone/version.rb +3 -0
  42. data/subdomains.lst +8214 -0
  43. data/templates/default.html.erb +225 -0
  44. metadata +159 -0
@@ -0,0 +1,187 @@
1
+ module Aquatone
2
+ module Commands
3
+ class Discover < Aquatone::Command
4
+ def execute!
5
+ if !options[:domain]
6
+ output("Please specify a domain to assess\n")
7
+ exit 1
8
+ end
9
+
10
+ @domain = Aquatone::Domain.new(options[:domain])
11
+ @assessment = Aquatone::Assessment.new(options[:domain])
12
+ @hosts = [options[:domain]]
13
+ @host_dictionary = {}
14
+
15
+ banner("Discover")
16
+ setup_resolver
17
+ identify_wildcard_ips
18
+ run_collectors
19
+ resolve_hosts
20
+ output_summary
21
+ write_to_hosts_file
22
+ rescue Aquatone::Domain::UnresolvableDomain => e
23
+ output(red("Error: #{e.message}\n"))
24
+ end
25
+
26
+ private
27
+
28
+ def setup_resolver
29
+ if options[:nameservers]
30
+ nameservers = options[:nameservers]
31
+ else
32
+ output("Identifying nameservers for #{@domain.name}... ")
33
+ nameservers = @domain.nameservers
34
+ output("Done\n")
35
+ if nameservers.empty?
36
+ output(yellow("#{@domain.name} has no nameservers. Using fallback nameservers.\n\n"))
37
+ nameservers = []
38
+ end
39
+ end
40
+
41
+ if !nameservers.empty?
42
+ output("Using nameservers:\n\n")
43
+ nameservers.each do |ns|
44
+ output(" - #{ns}\n")
45
+ end
46
+ output("\n")
47
+ end
48
+ @resolver = Aquatone::Resolver.new(
49
+ :nameservers => nameservers,
50
+ :fallback_nameservers => options[:fallback_nameservers]
51
+ )
52
+ end
53
+
54
+ def identify_wildcard_ips
55
+ output("Checking for wildcard DNS... ")
56
+ @wildcard_ips = []
57
+ wildcard_domain = "#{random_string}.#{@domain.name}"
58
+ if @resolver.resolve(wildcard_domain).nil?
59
+ output("Done\n")
60
+ return
61
+ end
62
+ output(yellow("Wildcard detected!\n"))
63
+ output("Identifying wildcard IPs... ")
64
+ 20.times do
65
+ wildcard_domain = "#{random_string}.#{@domain.name}"
66
+ if wildcard_ip = @resolver.resolve(wildcard_domain)
67
+ @wildcard_ips << wildcard_ip unless @wildcard_ips.include?(wildcard_ip)
68
+ end
69
+ end
70
+ output("Done\n")
71
+ output("Filtering out hosts resolving to wildcard IPs\n")
72
+ end
73
+
74
+ def run_collectors
75
+ output("\n")
76
+ Aquatone::Collector.descendants.each do |collector|
77
+ next if skip_collector?(collector)
78
+ output("Running collector: #{bold(collector.meta[:name])}... ")
79
+ begin
80
+ collector_instance = collector.new(@domain)
81
+ hosts = collector_instance.execute!
82
+ output("Done (#{hosts.count} #{hosts.count == 1 ? 'host' : 'hosts'})\n")
83
+ @hosts += hosts
84
+ rescue Aquatone::Collector::MissingKeyRequirement => e
85
+ output(yellow("Skipped\n"))
86
+ output(yellow(" -> #{e.message}\n"))
87
+ rescue => e
88
+ output(red("Error\n"))
89
+ output(red(" -> #{e.message}\n"))
90
+ end
91
+ end
92
+ @hosts = @hosts.sort.uniq
93
+ end
94
+
95
+ def resolve_hosts
96
+ output("\nResolving #{bold(@hosts.count)} unique hosts...\n")
97
+ task_count = 0
98
+ @start_time = Time.now.to_i
99
+ @hosts.each do |host|
100
+ if asked_for_progress?
101
+ output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
102
+ "#{task_count} out of #{@hosts.count} hosts checked (#{@host_dictionary.keys.count} discovered); " \
103
+ "#{(task_count.to_f / @hosts.count.to_f * 100.00).round(1)}% done\n")
104
+ end
105
+ if ip = @resolver.resolve(host)
106
+ next if wildcard_ip?(ip)
107
+ next if (options[:ignore_private] && private_ip?(ip))
108
+ @host_dictionary[host] = ip
109
+ output("#{ip.ljust(15)} #{bold(host)}\n")
110
+ end
111
+ jitter_sleep
112
+ task_count += 1
113
+ end
114
+ output("\n", true)
115
+ end
116
+
117
+ def output_summary
118
+ subnets = find_subnets
119
+ if !subnets.keys.count.zero?
120
+ output("Found subnets:\n\n")
121
+ subnets.each_pair do |subnet, count|
122
+ next if count == 1
123
+ subnet = "#{subnet}.0-255"
124
+ output(" - #{subnet.ljust(17)} : #{count} hosts\n")
125
+ end
126
+ end
127
+ output("\n")
128
+ end
129
+
130
+ def find_subnets
131
+ subnets = {}
132
+ @host_dictionary.values.each do |ip|
133
+ subnet = ip.split(".")[0..2].join(".")
134
+ if subnets.key?(subnet)
135
+ subnets[subnet] += 1
136
+ else
137
+ subnets[subnet] = 1
138
+ end
139
+ end
140
+ Hash[subnets.sort_by{|k, v| v}.reverse]
141
+ end
142
+
143
+ def write_to_hosts_file
144
+ @hosts_file_contents = ""
145
+ @host_dictionary.each_pair do |host, ip|
146
+ @hosts_file_contents += "#{host},#{ip}\n"
147
+ end
148
+ @assessment.write_file("hosts.txt", @hosts_file_contents)
149
+ @assessment.write_file("hosts.json", @host_dictionary.to_json)
150
+ output("Wrote #{bold(@host_dictionary.keys.count)} hosts to:\n\n")
151
+ output(" - #{bold('file://' + File.join(@assessment.path, 'hosts.txt'))}\n")
152
+ output(" - #{bold('file://' + File.join(@assessment.path, 'hosts.json'))}\n")
153
+ end
154
+
155
+ def random_string
156
+ %w(a b c d e f g h i j k l m n o p q r s t u v w x y z
157
+ 0 1 2 3 4 5 6 7 8 9).shuffle.take(10).join
158
+ end
159
+
160
+ def wildcard_ip?(ip)
161
+ @wildcard_ips.include?(ip)
162
+ end
163
+
164
+ def private_ip?(ip)
165
+ ip =~ /(\A127\.)|(\A10\.)|(\A172\.1[6-9]\.)|(\A172\.2[0-9]\.)|(\A172\.3[0-1]\.)|(\A192\.168\.)/
166
+ end
167
+
168
+ def skip_collector?(collector)
169
+ if options[:only_collectors]
170
+ if options[:only_collectors].include?(collector.sluggified_name)
171
+ false
172
+ else
173
+ true
174
+ end
175
+ elsif options[:disable_collectors]
176
+ if options[:disable_collectors].include?(collector.sluggified_name)
177
+ true
178
+ else
179
+ false
180
+ end
181
+ else
182
+ false
183
+ end
184
+ end
185
+ end
186
+ end
187
+ end
@@ -0,0 +1,167 @@
1
+ module Aquatone
2
+ module Commands
3
+ class Gather < Aquatone::Command
4
+ def execute!
5
+ if !options[:domain]
6
+ output("Please specify a domain to assess\n")
7
+ exit 1
8
+ end
9
+
10
+ @assessment = Aquatone::Assessment.new(options[:domain])
11
+
12
+ banner("Gather")
13
+ check_prerequisites
14
+ prepare_tasks
15
+ make_directories
16
+ process_pages
17
+ generate_report
18
+ end
19
+
20
+ private
21
+
22
+ def check_prerequisites
23
+ if !has_executable?("node")
24
+ output(red("node executable not found!\n\n"))
25
+ output("Please make sure Node.js is installed on your system.\n")
26
+ exit 1
27
+ end
28
+
29
+ if !has_executable?("npm")
30
+ output(red("npm executable not found!\n\n"))
31
+ output("Please make sure NPM package manager is installed on your system.\n")
32
+ exit 1
33
+ end
34
+
35
+ if !Dir.exists?(File.join(Aquatone::AQUATONE_ROOT, "node_modules"))
36
+ output("Installing Nightmare.js package, please wait...")
37
+ Dir.chdir(Aquatone::AQUATONE_ROOT) do
38
+ if system("npm install nightmare >/dev/null 2>&1")
39
+ output(" Done\n\n")
40
+ else
41
+ output(red(" Failed\n"))
42
+ exit 1
43
+ end
44
+ end
45
+ end
46
+ end
47
+
48
+ def prepare_tasks
49
+ if !@assessment.has_file?("hosts.json")
50
+ output(red("#{@assessment.path} does not contain hosts.json file\n\n"))
51
+ output("Did you run aquatone-discover first?\n")
52
+ exit 1
53
+ end
54
+ if !@assessment.has_file?("open_ports.txt")
55
+ output(red("#{@assessment.path} does not contain open_ports.txt file\n\n"))
56
+ output("Did you run aquatone-scan first?\n")
57
+ exit 1
58
+ end
59
+ @tasks = []
60
+ @hosts = JSON.parse(@assessment.read_file("hosts.json"))
61
+ @open_ports = parse_open_ports_file
62
+ @hosts.each_pair do |domain, host|
63
+ next unless @open_ports.key?(host)
64
+ @open_ports[host].each do |port|
65
+ @tasks << [host, port, domain]
66
+ end
67
+ end
68
+ end
69
+
70
+ def process_pages
71
+ pool = thread_pool
72
+ @task_count = 0
73
+ @successful = 0
74
+ @failed = 0
75
+ @start_time = Time.now.to_i
76
+ @visits = []
77
+ output("Processing #{bold(@tasks.count)} pages...\n")
78
+ @tasks.shuffle.each do |task|
79
+ host, port, domain = task
80
+ pool.schedule do
81
+ output_progress if asked_for_progress?
82
+ visit = visit_page(host, port, domain)
83
+ if visit['success']
84
+ output("#{green('Processed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['status']}\n")
85
+ @successful += 1
86
+ else
87
+ output(" #{red('Failed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['error']} #{visit['details']}\n")
88
+ @failed += 1
89
+ end
90
+ jitter_sleep
91
+ @task_count += 1
92
+ end
93
+ end
94
+ pool.shutdown
95
+ output("\nFinished processing pages:\n\n")
96
+ output(" - Successful : #{bold(green(@successful))}\n")
97
+ output(" - Failed : #{bold(red(@failed))}\n\n")
98
+ end
99
+
100
+ def generate_report
101
+ output("Generating report...")
102
+ report = Aquatone::Report.new(options[:domain], @visits)
103
+ report.generate(File.join(@assessment.path, "report"))
104
+ output("done\n")
105
+ report_pages = Dir[File.join(@assessment.path, "report", "report_page_*.html")]
106
+ output("Report pages generated:\n\n")
107
+ sort_report_pages(report_pages).each do |report_page|
108
+ output(" - file://#{report_page}\n")
109
+ end
110
+ output("\n")
111
+ end
112
+
113
+ def parse_open_ports_file
114
+ contents = @assessment.read_file("open_ports.txt")
115
+ result = {}
116
+ lines = contents.split("\n").map(&:strip)
117
+ lines.each do |line|
118
+ values = line.split(",").map(&:strip)
119
+ result[values[0]] = values[1..-1].map(&:to_i)
120
+ end
121
+ result
122
+ end
123
+
124
+ def make_directories
125
+ @assessment.make_directory("headers")
126
+ @assessment.make_directory("html")
127
+ @assessment.make_directory("report")
128
+ @assessment.make_directory("screenshots")
129
+ end
130
+
131
+ def make_file_basename(host, port, domain)
132
+ "#{domain}__#{host}__#{port}".downcase.gsub(".", "_")
133
+ end
134
+
135
+ def output_progress
136
+ output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
137
+ "#{@task_count} out of #{@tasks.count} pages processed (#{@successful} successful, #{@failed} failed); " \
138
+ "#{(@task_count.to_f / @tasks.count.to_f * 100.00).round(1)}% done\n")
139
+ end
140
+
141
+ def sort_report_pages(pages)
142
+ pages.sort_by { |f| File.basename(f).split("_").last.split(".").first.to_i }
143
+ end
144
+
145
+ def visit_page(host, port, domain)
146
+ file_basename = make_file_basename(host, port, domain)
147
+ url = Aquatone::UrlMaker.make(host, port)
148
+ html_destination = File.join(@assessment.path, "html", "#{file_basename}.html")
149
+ headers_destination = File.join(@assessment.path, "headers", "#{file_basename}.txt")
150
+ screenshot_destination = File.join(@assessment.path, "screenshots", "#{file_basename}.png")
151
+ visit = Aquatone::Browser.visit(url, domain, html_destination, headers_destination, screenshot_destination, :timeout => options[:timeout])
152
+ if visit['success']
153
+ @visits.push({
154
+ :host => host,
155
+ :port => port,
156
+ :domain => domain,
157
+ :url => url,
158
+ :file_basename => file_basename,
159
+ :headers => visit['headers'],
160
+ :status => visit['status']
161
+ })
162
+ end
163
+ visit
164
+ end
165
+ end
166
+ end
167
+ end
@@ -0,0 +1,108 @@
1
+ module Aquatone
2
+ module Commands
3
+ class Scan < Aquatone::Command
4
+ def execute!
5
+ if !options[:domain]
6
+ output("Please specify a domain to assess\n")
7
+ exit 1
8
+ end
9
+
10
+ @assessment = Aquatone::Assessment.new(options[:domain])
11
+ @tasks = []
12
+ @host_dictionary = {}
13
+ @results = {}
14
+ @urls = []
15
+
16
+ banner("Scan")
17
+ prepare_host_dictionary
18
+ scan_ports
19
+ write_open_ports_file
20
+ write_urls_file
21
+ end
22
+
23
+ def prepare_host_dictionary
24
+ if !@assessment.has_file?("hosts.json")
25
+ output(red("#{@assessment.path} does not contain hosts.json file\n\n"))
26
+ output("Did you run aquatone-discover first?\n")
27
+ exit 1
28
+ end
29
+ hosts = JSON.parse(@assessment.read_file("hosts.json"))
30
+ output("Loaded #{bold(hosts.count)} hosts from #{bold(File.join(@assessment.path, 'hosts.json'))}\n\n")
31
+ hosts.each_pair do |domain, ip|
32
+ if !@host_dictionary.key?(ip)
33
+ @host_dictionary[ip] = [domain]
34
+ options[:ports].each do |port|
35
+ @tasks << [ip, port]
36
+ end
37
+ else
38
+ @host_dictionary[ip] << domain
39
+ end
40
+ end
41
+ end
42
+
43
+ def scan_ports
44
+ pool = thread_pool
45
+ @task_count = 1
46
+ @ports_open = 0
47
+ @start_time = Time.now.to_i
48
+ output("Probing #{bold(@tasks.count)} ports...\n")
49
+ @tasks.shuffle.each do |task|
50
+ host, port = task
51
+ pool.schedule do
52
+ output_progress if asked_for_progress?
53
+ if port_open?(host, port)
54
+ output_open_port(host, port)
55
+ @ports_open += 1
56
+ @results[host] ||= []
57
+ @results[host] << port
58
+ @host_dictionary[host].each do |hostname|
59
+ @urls << Aquatone::UrlMaker.make(hostname, port)
60
+ end
61
+ end
62
+ jitter_sleep
63
+ @task_count += 1
64
+ end
65
+ end
66
+ pool.shutdown
67
+ end
68
+
69
+ def write_open_ports_file
70
+ contents = []
71
+ @results.each_pair do |host, ports|
72
+ contents << "#{host},#{ports.join(',')}"
73
+ end
74
+ @assessment.write_file("open_ports.txt", contents.sort.join("\n"))
75
+ output("\nWrote open ports to #{bold('file://' + File.join(@assessment.path, 'open_ports.txt'))}\n")
76
+ end
77
+
78
+ def write_urls_file
79
+ @assessment.write_file("urls.txt", @urls.uniq.sort.join("\n"))
80
+ output("Wrote URLs to #{bold('file://' + File.join(@assessment.path, 'urls.txt'))}\n")
81
+ end
82
+
83
+ def port_open?(host, port)
84
+ Timeout::timeout(options[:timeout]) do
85
+ TCPSocket.new(host, port).close
86
+ true
87
+ end
88
+ rescue Timeout::Error, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError
89
+ false
90
+ end
91
+
92
+ def output_progress
93
+ output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
94
+ "#{@task_count} out of #{@tasks.count} ports checked (#{@ports_open} open); " \
95
+ "#{(@task_count.to_f / @tasks.count.to_f * 100.00).round(1)}% done\n")
96
+ end
97
+
98
+ def output_open_port(host, port)
99
+ if (@host_dictionary[host].count > 3)
100
+ domains = @host_dictionary[host].shuffle.take(3).join(", ") + " and #{@host_dictionary[host].count - 3} more"
101
+ else
102
+ domains = @host_dictionary[host].take(3).join(", ")
103
+ end
104
+ output("#{green((port.to_s + '/tcp').ljust(9))} #{host.ljust(15)} #{domains}\n")
105
+ end
106
+ end
107
+ end
108
+ end