aquatone 0.1.0

data/lib/aquatone/collector.rb

@@ -0,0 +1,106 @@
+module Aquatone
+  class Collector
+    class Error < StandardError; end
+    class InvalidMetadataError < Error; end
+    class MetadataNotSetError < Error; end
+    class MissingKeyRequirement < Error; end
+
+    attr_reader :domain, :hosts
+
+    DEFAULT_PRIORITY = 1
+
+    def self.meta
+      @meta || fail(MetadataNotSetError, "Metadata has not been set")
+    end
+
+    def self.meta=(meta)
+      validate_metadata(meta)
+      @meta = meta
+    end
+
+    def self.descendants
+      collectors = ObjectSpace.each_object(Class).select { |klass| klass < self }
+      collectors.sort { |x, y| x.priority <=> y.priority }
+    end
+
+    def self.sluggified_name
+      return meta[:slug].downcase if meta[:slug]
+      meta[:name].strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub("--", "-")
+    end
+
+    def initialize(domain)
+      check_key_requirements!
+      @domain = domain
+      @hosts  = []
+    end
+
+    def run
+      fail NotImplementedError
+    end
+
+    def execute!
+      run
+      hosts
+    end
+
+    def self.priority
+      meta[:priority] || DEFAULT_PRIORITY
+    end
+
+    protected
+
+    def add_host(host)
+      host.downcase!
+      return unless Aquatone::Validation.valid_domain_name?(host)
+      @hosts << host unless @hosts.include?(host)
+    end
+
+    def get_request(uri, options={})
+      Aquatone::HttpClient.get(uri)
+    end
+
+    def post_request(uri, body=nil, options={})
+      options = {
+        :body => body
+      }.merge(options)
+      Aquatone::HttpClient.post(uri, options)
+    end
+
+    def url_escape(string)
+      CGI.escape(string)
+    end
+
+    def random_sleep(seconds)
+      random_sleep = ((1 - (rand(30) * 0.01)) * seconds.to_i)
+      sleep(random_sleep)
+    end
+
+    def get_key(name)
+      Aquatone::KeyStore.get(name)
+    end
+
+    def has_key?(name)
+      Aquatone::KeyStore.key?(name)
+    end
+
+    def failure(message)
+      fail Error, message
+    end
+
+    def check_key_requirements!
+      return unless self.class.meta[:require_keys]
+      keys = self.class.meta[:require_keys]
+      keys.each do |key|
+        fail MissingKeyRequirement, "Key '#{key}' has not been set" unless has_key?(key)
+      end
+    end
+
+    def self.validate_metadata(meta)
+      fail InvalidMetadataError, "Metadata is not a hash" unless meta.is_a?(Hash)
+      fail InvalidMetadataError, "Metadata is empty" if meta.empty?
+      fail InvalidMetadataError, "Metadata is missing key: name" unless meta.key?(:name)
+      fail InvalidMetadataError, "Metadata is missing key: author" unless meta.key?(:author)
+      fail InvalidMetadataError, "Metadata is missing key: description" unless meta.key?(:description)
+    end
+  end
+end

data/lib/aquatone/collectors/dictionary.rb

@@ -0,0 +1,20 @@
+module Aquatone
+  module Collectors
+    class Dictionary < Aquatone::Collector
+      self.meta = {
+        :name        => "Dictionary",
+        :author      => "Michael Henriksen (@michenriksen)",
+        :description => "Uses a dictionary to find hostnames"
+      }
+
+      DICTIONARY = File.join(Aquatone::AQUATONE_ROOT, "subdomains.lst").freeze
+
+      def run
+        dictionary = File.open(DICTIONARY, "r")
+        dictionary.each_line do |subdomain|
+          add_host("#{subdomain.strip}.#{domain.name}")
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/dnsdb.rb

@@ -0,0 +1,45 @@
+module Aquatone
+  module Collectors
+    class Dnsdb < Aquatone::Collector
+      self.meta = {
+        :name        => "DNSDB",
+        :author      => "Michael Henriksen (@michenriksen)",
+        :description => "Uses dnsdb.org to find hostnames"
+      }
+
+      LINK_REGEX = /<a href="(.*?)\">(.*?)<\/a>/.freeze
+      INDEX_REGEX = /<a href="([0-9a-z]{1})">[0-9a-z]{1}<\/a>/.freeze
+
+      def run
+        @base_url = "http://www.dnsdb.org/f/#{url_escape(domain.name)}.dnsdb.org/"
+        parse_page(@base_url)
+      end
+
+      private
+
+      def parse_page(url)
+        response = get_request(url)
+        if response.code != 200
+          failure("DNSDB returned unexpected response code: #{response.code}")
+        end
+
+        if response.body.include?("index for")
+          response.body.scan(INDEX_REGEX) do |index|
+            response = get_request("#{@base_url}#{url_escape(index.first)}")
+            extract_hosts(response.body)
+          end
+        else
+          extract_hosts(response.body)
+        end
+      end
+
+      def extract_hosts(body)
+        body.scan(LINK_REGEX) do |href, hostname|
+          if hostname.end_with?(".#{domain.name}")
+            add_host(hostname)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/gtr.rb

@@ -0,0 +1,58 @@
+module Aquatone
+  module Collectors
+    class Gtr < Aquatone::Collector
+      self.meta = {
+        :name        => "Google Transparency Report",
+        :author      => "Michael Henriksen (@michenriksen)",
+        :description => "Uses Google Transparency Report to find hostnames",
+        :slug        => "gtr"
+      }
+
+      BASE_URI         = "https://www.google.com/transparencyreport/jsonp/ct/search"
+      PAGES_TO_PROCESS = 30.freeze
+
+      def run
+        token = nil
+        PAGES_TO_PROCESS.times do
+          response = parse_response(request_page(token))
+          response["results"].each do |result|
+            host = result["subject"]
+            add_host(host) if valid_host?(host)
+          end
+          break if !response.key?("nextPageToken")
+          token = response["nextPageToken"]
+        end
+      end
+
+      private
+
+      def request_page(token = nil)
+        if token.nil?
+          uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&incl_exp=true&incl_sub=true&c=_callbacks_._#{random_jsonp_callback}"
+        else
+          uri = "#{BASE_URI}?domain=#{url_escape(domain.name)}&incl_exp=true&incl_sub=true&token=#{url_escape(token)}&c=_callbacks_._#{random_jsonp_callback}"
+        end
+
+        get_request(uri,
+          { :headers => { "Referer" => "https://www.google.com/transparencyreport/https/ct/?hl=en-US" } }
+        )
+      end
+
+      def random_jsonp_callback
+        "abcdefghijklmnopqrstuvwxyz0123456789".split("").sample(9).join
+      end
+
+      def parse_response(body)
+        body = body.split("(", 2).last
+        body.gsub!(");", "")
+        JSON.parse(body)
+      end
+
+      def valid_host?(host)
+        return false if host.start_with?("*.")
+        return false unless host.end_with?(".#{domain.name}")
+        true
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/hackertarget.rb

@@ -0,0 +1,24 @@
+module Aquatone
+  module Collectors
+    class Hackertarget < Aquatone::Collector
+      self.meta = {
+        :name        => "HackerTarget",
+        :author      => "Michael Henriksen (@michenriksen)",
+        :description => "Uses hackertarget.com to find hostnames"
+      }
+
+      API_BASE_URI = "https://api.hackertarget.com"
+
+      def run
+        response = get_request("#{API_BASE_URI}/hostsearch/?q=#{url_escape(domain.name)}")
+        if response.code != 200
+          failure("HackerTarget API returned unexpected response code: #{response.code}")
+        end
+        response.body.each_line do |line|
+          host = line.split(",", 2).first.strip
+          add_host(host) unless host.empty?
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/netcraft.rb

@@ -0,0 +1,48 @@
+module Aquatone
+  module Collectors
+    class Netcraft < Aquatone::Collector
+      self.meta = {
+        :name        => "Netcraft",
+        :author      => "Michael Henriksen (@michenriksen)",
+        :description => "Uses searchdns.netcraft.com to find hostnames"
+      }
+
+      BASE_URI         = "http://searchdns.netcraft.com/".freeze
+      HOSTNAME_REGEX   = /<a href="http:\/\/(.*?)\/" rel="nofollow">/.freeze
+      RESULTS_PER_PAGE = 20.freeze
+      PAGES_TO_PROCESS = 10.freeze
+
+      def run
+        last  = nil
+        count = 0
+        PAGES_TO_PROCESS.times do |i|
+          page = i + 1
+          if page == 1
+            uri = "#{BASE_URI}/?restriction=site+contains&host=*.#{url_escape(domain.name)}&lookup=wait..&position=limited"
+          else
+            uri = "#{BASE_URI}/?host=*.#{url_escape(domain.name)}&last=#{url_escape(last)}&from=#{count + 1}&restriction=site%20contains&position=limited"
+          end
+          response = get_request(uri,
+            { :headers => { "Referer" => "http://searchdns.netcraft.com/" } }
+          )
+          hosts = extract_hostnames_from_response(response.body)
+          last  = hosts.last
+          count += hosts.count
+          hosts.each { |host| add_host(host) }
+          break if hosts.count != RESULTS_PER_PAGE
+          random_sleep(5)
+        end
+      end
+
+      private
+
+      def extract_hostnames_from_response(body)
+        hosts = []
+        body.scan(HOSTNAME_REGEX).each do |match|
+          hosts << match.last.to_s.strip.downcase
+        end
+        hosts
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/shodan.rb

@@ -0,0 +1,45 @@
+module Aquatone
+  module Collectors
+    class Shodan < Aquatone::Collector
+      self.meta = {
+        :name         => "Shodan",
+        :author       => "Michael Henriksen (@michenriksen)",
+        :description  => "Uses the Shodan API to find hostnames",
+        :require_keys => ["shodan"]
+      }
+
+      API_BASE_URI         = "https://api.shodan.io/shodan".freeze
+      API_RESULTS_PER_PAGE = 100.freeze
+      PAGES_TO_PROCESS     = 10.freeze
+
+      def run
+        request_shodan_page
+      end
+
+      private
+
+      def request_shodan_page(page=1)
+        response = get_request(construct_uri("hostname:#{domain.name}", page))
+        if response.code != 200
+          failure(response.parsed_response["error"] || "Shodan API returned unexpected response code: #{response.code}")
+        end
+        return unless response.parsed_response["matches"]
+        response.parsed_response["matches"].each do |match|
+          next unless match["hostnames"]
+          match["hostnames"].each do |hostname|
+            add_host(hostname) if hostname.end_with?(".#{domain.name}")
+          end
+        end
+        request_shodan_page(page + 1) if next_page?(page, response.parsed_response)
+      end
+
+      def construct_uri(query, page)
+        "#{API_BASE_URI}/host/search?query=#{url_escape(query)}&page=#{page}&key=#{get_key('shodan')}"
+      end
+
+      def next_page?(page, body)
+        page <= PAGES_TO_PROCESS && body["total"] && API_RESULTS_PER_PAGE * page < body["total"].to_i
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/threatcrowd.rb

@@ -0,0 +1,25 @@
+module Aquatone
+  module Collectors
+    class Threatcrowd < Aquatone::Collector
+      self.meta = {
+        :name         => "Threat Crowd",
+        :author       => "Michael Henriksen (@michenriksen)",
+        :description  => "Uses threadcrowd.org API to find hostnames",
+        :slug         => "threatcrowd"
+      }
+
+      API_URI = "https://www.threatcrowd.org/searchApi/v2/domain/report/".freeze
+
+      def run
+        response = get_request("#{API_URI}?domain=#{url_escape(domain.name)}")
+        if response.code != 200
+          failure("Threat Crowd API returned unexpected status code: #{response.code}")
+        end
+        body = JSON.parse(response.body)
+        if body.key?("subdomains")
+          body["subdomains"].each { |host| add_host(host) }
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/virustotal.rb

@@ -0,0 +1,24 @@
+module Aquatone
+  module Collectors
+    class Virustotal < Aquatone::Collector
+      self.meta = {
+        :name         => "VirusTotal",
+        :author       => "Michael Henriksen (@michenriksen)",
+        :description  => "Uses virustotal.com domain search to find hostnames",
+        :require_keys => ["virustotal"]
+      }
+
+      API_URI = "http://www.virustotal.com/vtapi/v2/domain/report".freeze
+
+      def run
+        response = get_request("#{API_URI}?domain=#{url_escape(domain.name)}&apikey=#{get_key('virustotal')}")
+        if response.code != 200
+          failure("VirusTotal API returned unexpected status code: #{response.code}")
+        end
+        if response.parsed_response.key?("subdomains")
+          response.parsed_response["subdomains"].each { |host| add_host(host) }
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/command.rb

@@ -0,0 +1,152 @@
+module Aquatone
+  class Command
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def self.run(options)
+      self.new(options).execute!
+    rescue Interrupt
+      output("Caught interrupt; exiting.\n", true)
+    rescue => e
+      output(red("An unexpected error occurred: #{e.class}: #{e.message}\n"))
+      output("#{e.backtrace.join("\n")}\n")
+    end
+
+    def execute!
+      fail NotImplementedError, "Commands must overwrite #execute! method"
+    end
+
+    protected
+
+    def output(text, clear_line = false)
+      self.class.output(text, clear_line)
+    end
+
+    def status(message)
+      self.class.status(message)
+    end
+
+    def self.output(text, clear_line = false)
+      if clear_line
+        text = "\r\e[0K#{text}"
+      end
+      with_output_mutex { print text }
+    end
+
+    def self.status(message)
+      output(message, true)
+    end
+
+    def self.colorize(text, color_code)
+      "\e[1m\e[#{color_code}m#{text}\e[0m"
+    end
+
+    def uncolorize(text)
+      self.class.uncolorize(text)
+    end
+
+    def red(text)
+      self.class.red(text)
+    end
+
+    def green(text)
+      self.class.green(text)
+    end
+
+    def blue(text)
+      self.class.blue(text)
+    end
+
+    def yellow(text)
+      self.class.yellow(text)
+    end
+
+    def bold(text)
+      self.class.bold(text)
+    end
+
+    def banner(subtitle)
+      output("                           __\n")
+      output("  ____ _____ ___  ______ _/ /_____  ____  ___\n")
+      output(" / __ `/ __ `/ / / / __ `/ __/ __ \\/ __ \\/ _ \\\n")
+      output("/ /_/ / /_/ / /_/ / /_/ / /_/ /_/ / / / /  __/\n")
+      output("\\__,_/\\__, /\\__,_/\\__,_/\\__/\\____/_/ /_/\\___/\n")
+      output("        /_/  #{subtitle.downcase} v#{Aquatone::VERSION} - by @michenriksen\n\n")
+    end
+
+    def self.uncolorize(text); colorize(text.to_s, 0); end
+    def self.red(text); colorize(text.to_s, 31); end
+    def self.green(text); colorize(text.to_s, 32); end
+    def self.blue(text); colorize(text.to_s, 34); end
+    def self.yellow(text); colorize(text.to_s, 33); end
+    def self.bold(text); colorize(text.to_s, 1); end
+
+    def thread_pool
+      if options[:sleep]
+        Aquatone::ThreadPool.new(1)
+      else
+        Aquatone::ThreadPool.new(options[:threads])
+      end
+    end
+
+    def jitter_sleep
+      return unless options[:sleep]
+      seconds = options[:sleep].to_i
+      if options[:jitter]
+        jitter  = (options[:jitter].to_f / 100) * seconds
+        if rand.round == 0
+          seconds = seconds - Random.rand(0..jitter.round)
+        else
+          seconds = seconds + Random.rand(0..jitter.round)
+        end
+        seconds = 1 if seconds < 1
+      end
+      sleep seconds
+    end
+
+    def seconds_to_time(seconds)
+      Time.at(seconds).utc.strftime("%H:%M:%S")
+    end
+
+    def asked_for_progress?
+      while c = STDIN.read_nonblock(1)
+        return true if c == "\n"
+      end
+      false
+    rescue IO::EAGAINWaitReadable
+      false
+    rescue Errno::EINTR, Errno::EAGAIN, EOFError
+      true
+    end
+
+    def has_executable?(name)
+      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+      ENV["PATH"].split(File::PATH_SEPARATOR).each do |path|
+        exts.each do |ext|
+          exe = File.join(path, "#{name}#{ext}")
+          return true if File.executable?(exe) && !File.directory?(exe)
+        end
+      end
+      false
+    end
+
+    def self.jitter
+      seconds = @options[:jitter]
+      if seconds != 0
+        random_sleep = ((1 - (rand(30) * 0.01)) * seconds)
+        sleep(random_sleep)
+      end
+    end
+
+    def self.with_output_mutex
+      output_mutex.synchronize { yield }
+    end
+
+    def self.output_mutex
+      @output_mutex ||= Mutex.new
+    end
+  end
+end