aquatone 0.1.0

data/lib/aquatone/commands/discover.rb

@@ -0,0 +1,187 @@
+module Aquatone
+  module Commands
+    class Discover < Aquatone::Command
+      def execute!
+        if !options[:domain]
+          output("Please specify a domain to assess\n")
+          exit 1
+        end
+
+        @domain          = Aquatone::Domain.new(options[:domain])
+        @assessment      = Aquatone::Assessment.new(options[:domain])
+        @hosts           = [options[:domain]]
+        @host_dictionary = {}
+
+        banner("Discover")
+        setup_resolver
+        identify_wildcard_ips
+        run_collectors
+        resolve_hosts
+        output_summary
+        write_to_hosts_file
+      rescue Aquatone::Domain::UnresolvableDomain => e
+        output(red("Error: #{e.message}\n"))
+      end
+
+      private
+
+      def setup_resolver
+        if options[:nameservers]
+          nameservers = options[:nameservers]
+        else
+          output("Identifying nameservers for #{@domain.name}... ")
+          nameservers = @domain.nameservers
+          output("Done\n")
+          if nameservers.empty?
+            output(yellow("#{@domain.name} has no nameservers. Using fallback nameservers.\n\n"))
+            nameservers = []
+          end
+        end
+
+        if !nameservers.empty?
+          output("Using nameservers:\n\n")
+          nameservers.each do |ns|
+            output(" - #{ns}\n")
+          end
+          output("\n")
+        end
+        @resolver = Aquatone::Resolver.new(
+          :nameservers          => nameservers,
+          :fallback_nameservers => options[:fallback_nameservers]
+        )
+      end
+
+      def identify_wildcard_ips
+        output("Checking for wildcard DNS... ")
+        @wildcard_ips   = []
+        wildcard_domain = "#{random_string}.#{@domain.name}"
+        if @resolver.resolve(wildcard_domain).nil?
+          output("Done\n")
+          return
+        end
+        output(yellow("Wildcard detected!\n"))
+        output("Identifying wildcard IPs... ")
+        20.times do
+          wildcard_domain = "#{random_string}.#{@domain.name}"
+          if wildcard_ip = @resolver.resolve(wildcard_domain)
+            @wildcard_ips << wildcard_ip unless @wildcard_ips.include?(wildcard_ip)
+          end
+        end
+        output("Done\n")
+        output("Filtering out hosts resolving to wildcard IPs\n")
+      end
+
+      def run_collectors
+        output("\n")
+        Aquatone::Collector.descendants.each do |collector|
+          next if skip_collector?(collector)
+          output("Running collector: #{bold(collector.meta[:name])}... ")
+          begin
+            collector_instance = collector.new(@domain)
+            hosts = collector_instance.execute!
+            output("Done (#{hosts.count} #{hosts.count == 1 ? 'host' : 'hosts'})\n")
+            @hosts += hosts
+          rescue Aquatone::Collector::MissingKeyRequirement => e
+            output(yellow("Skipped\n"))
+            output(yellow(" -> #{e.message}\n"))
+          rescue => e
+            output(red("Error\n"))
+            output(red(" -> #{e.message}\n"))
+          end
+        end
+        @hosts = @hosts.sort.uniq
+      end
+
+      def resolve_hosts
+        output("\nResolving #{bold(@hosts.count)} unique hosts...\n")
+        task_count = 0
+        @start_time = Time.now.to_i
+        @hosts.each do |host|
+          if asked_for_progress?
+            output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
+                   "#{task_count} out of #{@hosts.count} hosts checked (#{@host_dictionary.keys.count} discovered); " \
+                   "#{(task_count.to_f / @hosts.count.to_f * 100.00).round(1)}% done\n")
+          end
+          if ip = @resolver.resolve(host)
+            next if wildcard_ip?(ip)
+            next if (options[:ignore_private] && private_ip?(ip))
+            @host_dictionary[host] = ip
+            output("#{ip.ljust(15)} #{bold(host)}\n")
+          end
+          jitter_sleep
+          task_count += 1
+        end
+        output("\n", true)
+      end
+
+      def output_summary
+        subnets = find_subnets
+        if !subnets.keys.count.zero?
+          output("Found subnets:\n\n")
+          subnets.each_pair do |subnet, count|
+            next if count == 1
+            subnet = "#{subnet}.0-255"
+            output(" - #{subnet.ljust(17)} : #{count} hosts\n")
+          end
+        end
+        output("\n")
+      end
+
+      def find_subnets
+        subnets = {}
+        @host_dictionary.values.each do |ip|
+          subnet = ip.split(".")[0..2].join(".")
+          if subnets.key?(subnet)
+            subnets[subnet] += 1
+          else
+            subnets[subnet] = 1
+          end
+        end
+        Hash[subnets.sort_by{|k, v| v}.reverse]
+      end
+
+      def write_to_hosts_file
+        @hosts_file_contents = ""
+        @host_dictionary.each_pair do |host, ip|
+          @hosts_file_contents += "#{host},#{ip}\n"
+        end
+        @assessment.write_file("hosts.txt", @hosts_file_contents)
+        @assessment.write_file("hosts.json", @host_dictionary.to_json)
+        output("Wrote #{bold(@host_dictionary.keys.count)} hosts to:\n\n")
+        output(" - #{bold('file://' + File.join(@assessment.path, 'hosts.txt'))}\n")
+        output(" - #{bold('file://' + File.join(@assessment.path, 'hosts.json'))}\n")
+      end
+
+      def random_string
+        %w(a b c d e f g h i j k l m n o p q r s t u v w x y z
+           0 1 2 3 4 5 6 7 8 9).shuffle.take(10).join
+      end
+
+      def wildcard_ip?(ip)
+        @wildcard_ips.include?(ip)
+      end
+
+      def private_ip?(ip)
+        ip =~ /(\A127\.)|(\A10\.)|(\A172\.1[6-9]\.)|(\A172\.2[0-9]\.)|(\A172\.3[0-1]\.)|(\A192\.168\.)/
+      end
+
+      def skip_collector?(collector)
+        if options[:only_collectors]
+          if options[:only_collectors].include?(collector.sluggified_name)
+            false
+          else
+            true
+          end
+        elsif options[:disable_collectors]
+          if options[:disable_collectors].include?(collector.sluggified_name)
+            true
+          else
+            false
+          end
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/commands/gather.rb

@@ -0,0 +1,167 @@
+module Aquatone
+  module Commands
+    class Gather < Aquatone::Command
+      def execute!
+        if !options[:domain]
+          output("Please specify a domain to assess\n")
+          exit 1
+        end
+
+        @assessment = Aquatone::Assessment.new(options[:domain])
+
+        banner("Gather")
+        check_prerequisites
+        prepare_tasks
+        make_directories
+        process_pages
+        generate_report
+      end
+
+      private
+
+      def check_prerequisites
+        if !has_executable?("node")
+          output(red("node executable not found!\n\n"))
+          output("Please make sure Node.js is installed on your system.\n")
+          exit 1
+        end
+
+        if !has_executable?("npm")
+          output(red("npm executable not found!\n\n"))
+          output("Please make sure NPM package manager is installed on your system.\n")
+          exit 1
+        end
+
+        if !Dir.exists?(File.join(Aquatone::AQUATONE_ROOT, "node_modules"))
+          output("Installing Nightmare.js package, please wait...")
+          Dir.chdir(Aquatone::AQUATONE_ROOT) do
+            if system("npm install nightmare >/dev/null 2>&1")
+              output(" Done\n\n")
+            else
+              output(red(" Failed\n"))
+              exit 1
+            end
+          end
+        end
+      end
+
+      def prepare_tasks
+        if !@assessment.has_file?("hosts.json")
+          output(red("#{@assessment.path} does not contain hosts.json file\n\n"))
+          output("Did you run aquatone-discover first?\n")
+          exit 1
+        end
+        if !@assessment.has_file?("open_ports.txt")
+          output(red("#{@assessment.path} does not contain open_ports.txt file\n\n"))
+          output("Did you run aquatone-scan first?\n")
+          exit 1
+        end
+        @tasks = []
+        @hosts = JSON.parse(@assessment.read_file("hosts.json"))
+        @open_ports = parse_open_ports_file
+        @hosts.each_pair do |domain, host|
+          next unless @open_ports.key?(host)
+          @open_ports[host].each do |port|
+            @tasks << [host, port, domain]
+          end
+        end
+      end
+
+      def process_pages
+        pool        = thread_pool
+        @task_count = 0
+        @successful = 0
+        @failed     = 0
+        @start_time = Time.now.to_i
+        @visits     = []
+        output("Processing #{bold(@tasks.count)} pages...\n")
+        @tasks.shuffle.each do |task|
+          host, port, domain = task
+          pool.schedule do
+            output_progress if asked_for_progress?
+            visit = visit_page(host, port, domain)
+            if visit['success']
+              output("#{green('Processed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['status']}\n")
+              @successful += 1
+            else
+              output("   #{red('Failed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['error']} #{visit['details']}\n")
+              @failed += 1
+            end
+            jitter_sleep
+            @task_count += 1
+          end
+        end
+        pool.shutdown
+        output("\nFinished processing pages:\n\n")
+        output(" - Successful : #{bold(green(@successful))}\n")
+        output(" - Failed     : #{bold(red(@failed))}\n\n")
+      end
+
+      def generate_report
+        output("Generating report...")
+        report = Aquatone::Report.new(options[:domain], @visits)
+        report.generate(File.join(@assessment.path, "report"))
+        output("done\n")
+        report_pages = Dir[File.join(@assessment.path, "report", "report_page_*.html")]
+        output("Report pages generated:\n\n")
+        sort_report_pages(report_pages).each do |report_page|
+          output(" - file://#{report_page}\n")
+        end
+        output("\n")
+      end
+
+      def parse_open_ports_file
+        contents = @assessment.read_file("open_ports.txt")
+        result   = {}
+        lines    = contents.split("\n").map(&:strip)
+        lines.each do |line|
+          values = line.split(",").map(&:strip)
+          result[values[0]] = values[1..-1].map(&:to_i)
+        end
+        result
+      end
+
+      def make_directories
+        @assessment.make_directory("headers")
+        @assessment.make_directory("html")
+        @assessment.make_directory("report")
+        @assessment.make_directory("screenshots")
+      end
+
+      def make_file_basename(host, port, domain)
+        "#{domain}__#{host}__#{port}".downcase.gsub(".", "_")
+      end
+
+      def output_progress
+        output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
+               "#{@task_count} out of #{@tasks.count} pages processed (#{@successful} successful, #{@failed} failed); " \
+               "#{(@task_count.to_f / @tasks.count.to_f * 100.00).round(1)}% done\n")
+      end
+
+      def sort_report_pages(pages)
+        pages.sort_by { |f| File.basename(f).split("_").last.split(".").first.to_i }
+      end
+
+      def visit_page(host, port, domain)
+        file_basename          = make_file_basename(host, port, domain)
+        url                    = Aquatone::UrlMaker.make(host, port)
+        html_destination       = File.join(@assessment.path, "html", "#{file_basename}.html")
+        headers_destination    = File.join(@assessment.path, "headers", "#{file_basename}.txt")
+        screenshot_destination = File.join(@assessment.path, "screenshots", "#{file_basename}.png")
+        visit = Aquatone::Browser.visit(url, domain, html_destination, headers_destination, screenshot_destination, :timeout => options[:timeout])
+        if visit['success']
+          @visits.push({
+            :host          => host,
+            :port          => port,
+            :domain        => domain,
+            :url           => url,
+            :file_basename => file_basename,
+            :headers       => visit['headers'],
+            :status        => visit['status']
+          })
+        end
+        visit
+      end
+    end
+  end
+end

data/lib/aquatone/commands/scan.rb

@@ -0,0 +1,108 @@
+module Aquatone
+  module Commands
+    class Scan < Aquatone::Command
+      def execute!
+        if !options[:domain]
+          output("Please specify a domain to assess\n")
+          exit 1
+        end
+
+        @assessment      = Aquatone::Assessment.new(options[:domain])
+        @tasks           = []
+        @host_dictionary = {}
+        @results         = {}
+        @urls            = []
+
+        banner("Scan")
+        prepare_host_dictionary
+        scan_ports
+        write_open_ports_file
+        write_urls_file
+      end
+
+      def prepare_host_dictionary
+        if !@assessment.has_file?("hosts.json")
+          output(red("#{@assessment.path} does not contain hosts.json file\n\n"))
+          output("Did you run aquatone-discover first?\n")
+          exit 1
+        end
+        hosts = JSON.parse(@assessment.read_file("hosts.json"))
+        output("Loaded #{bold(hosts.count)} hosts from #{bold(File.join(@assessment.path, 'hosts.json'))}\n\n")
+        hosts.each_pair do |domain, ip|
+          if !@host_dictionary.key?(ip)
+            @host_dictionary[ip] = [domain]
+            options[:ports].each do |port|
+              @tasks << [ip, port]
+            end
+          else
+            @host_dictionary[ip] << domain
+          end
+        end
+      end
+
+      def scan_ports
+        pool        = thread_pool
+        @task_count = 1
+        @ports_open = 0
+        @start_time = Time.now.to_i
+        output("Probing #{bold(@tasks.count)} ports...\n")
+        @tasks.shuffle.each do |task|
+          host, port = task
+          pool.schedule do
+            output_progress if asked_for_progress?
+            if port_open?(host, port)
+              output_open_port(host, port)
+              @ports_open += 1
+              @results[host] ||= []
+              @results[host] << port
+              @host_dictionary[host].each do |hostname|
+                @urls << Aquatone::UrlMaker.make(hostname, port)
+              end
+            end
+            jitter_sleep
+            @task_count += 1
+          end
+        end
+        pool.shutdown
+      end
+
+      def write_open_ports_file
+        contents = []
+        @results.each_pair do |host, ports|
+          contents << "#{host},#{ports.join(',')}"
+        end
+        @assessment.write_file("open_ports.txt", contents.sort.join("\n"))
+        output("\nWrote open ports to #{bold('file://' + File.join(@assessment.path, 'open_ports.txt'))}\n")
+      end
+
+      def write_urls_file
+        @assessment.write_file("urls.txt", @urls.uniq.sort.join("\n"))
+        output("Wrote URLs to #{bold('file://' + File.join(@assessment.path, 'urls.txt'))}\n")
+      end
+
+      def port_open?(host, port)
+        Timeout::timeout(options[:timeout]) do
+          TCPSocket.new(host, port).close
+          true
+        end
+      rescue Timeout::Error, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError
+        false
+      end
+
+      def output_progress
+        output("Stats: #{seconds_to_time(Time.now.to_i - @start_time)} elapsed; " \
+               "#{@task_count} out of #{@tasks.count} ports checked (#{@ports_open} open); " \
+               "#{(@task_count.to_f / @tasks.count.to_f * 100.00).round(1)}% done\n")
+      end
+
+      def output_open_port(host, port)
+        if (@host_dictionary[host].count > 3)
+          domains = @host_dictionary[host].shuffle.take(3).join(", ") + " and #{@host_dictionary[host].count - 3} more"
+        else
+          domains = @host_dictionary[host].take(3).join(", ")
+        end
+        output("#{green((port.to_s + '/tcp').ljust(9))} #{host.ljust(15)} #{domains}\n")
+      end
+    end
+  end
+end