aquatone 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89f0cbe4f00ec77bae2d323be2978261680a75d8
-  data.tar.gz: 32948fb248e6614400c878ea2ac4681a5da9558b
+  metadata.gz: d8ad3da3d450ecd7fd41c64cd0b18f0a3e3ae00b
+  data.tar.gz: 240fbce3dc353f146a32be28566201b2b62fd2c6
 SHA512:
-  metadata.gz: 941e0e136a451eb295184a6b35dde232cbcae72624a4e421a5c304f89f70c281744cc85c05eafe951f210fd744ec6b3406362b554a4cab9a46b666e58c7be254
-  data.tar.gz: 48ccd44ff3e7fa4cfd05bd7221e6287479f9b80b362342915789ffe673c12a9f8a25fc8ef63524286eee401f4fccddf0fb8395810ca1e352aa6b81c0445325df
+  metadata.gz: 244cc756b2cb1c65d36281fbeae199d290a71434f73a12c96e855995eff2a091aae442212d13ee604ac6b34d6d74196813ff3b01bd5b809b16512c86b438c44e
+  data.tar.gz: b109618403ee14536a67a8e75334e88279578f2eb9532edd8ea772c7bf1e443a8747a01038eadfc44bd5a42000ab90571be269432340eef1257161f6d3d40110

data/CHANGELOG.md

@@ -10,6 +10,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 
 
+## [0.4.0]
+### Added
+ - Collector module defined CLI options: Collectors can now define their own CLI options for `aquatone-discover`,
+   e.g. `--wordlist` to make the Dictionary collector use a custom wordlist instead of the built-in one.
+   See `aquatone-discover --help` for all new options.
+
+### Changed
+
+### Fixed
+ - Performance improvement in the way collector modules check for duplicate hosts (was only an issue with
+   very large results or dictionaries)
+
+
 ## [0.3.0]
 ### Added
  - New Tool: aquatone-takeover: Check discovered hosts for subdomain takeover vulnerabilities
@@ -47,7 +60,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
-[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.3.0...HEAD
+[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/michenriksen/aquatone/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/michenriksen/aquatone/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/michenriksen/aquatone/compare/v0.1.1...v0.2.0
 [0.1.1]: https://github.com/michenriksen/aquatone/compare/v0.1.0...v0.1.1

data/exe/aquatone-discover

@@ -121,6 +121,14 @@ OptionParser.new do |opts|
     options[:jitter] = v.to_f
   end
 
+  Aquatone::Collector.descendants.each do |collector|
+    collector.cli_options.each_pair do |option, description|
+      opts.on("--#{option}", description) do |v|
+        options[option.split(" ").first.gsub("-", "_").to_sym] = v
+      end
+    end
+  end
+
   opts.on("-h", "--help", "Show help") do
     puts opts
     exit 0

data/lib/aquatone/collector.rb

@@ -23,15 +23,21 @@ module Aquatone
       collectors.sort { |x, y| x.priority <=> y.priority }
     end
 
+    def self.cli_options
+      meta.key?(:cli_options) ? meta[:cli_options] : {}
+    end
+
     def self.sluggified_name
       return meta[:slug].downcase if meta[:slug]
       meta[:name].strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub("--", "-")
     end
 
-    def initialize(domain)
+    def initialize(domain, options)
       check_key_requirements!
-      @domain = domain
-      @hosts  = []
+      @domain          = domain
+      @options         = options
+      @hosts           = []
+      @host_dictionary = {}
     end
 
     def run
@@ -52,7 +58,9 @@ module Aquatone
     def add_host(host)
       host.downcase!
       return unless Aquatone::Validation.valid_domain_name?(host)
-      @hosts << host unless @hosts.include?(host)
+      return if @host_dictionary.key?(host)
+      @host_dictionary[host] = true
+      @hosts << host
     end
 
     def get_request(uri, options={})
@@ -83,6 +91,14 @@ module Aquatone
       Aquatone::KeyStore.key?(name)
     end
 
+    def get_cli_option(name)
+      @options[name.to_s.gsub("-", "_").to_sym]
+    end
+
+    def has_cli_option?(name)
+      @options.key?(name.to_s.gsub("-", "_").to_sym)
+    end
+
     def failure(message)
       fail Error, message
     end
@@ -101,6 +117,13 @@ module Aquatone
       fail InvalidMetadataError, "Metadata is missing key: name" unless meta.key?(:name)
       fail InvalidMetadataError, "Metadata is missing key: author" unless meta.key?(:author)
       fail InvalidMetadataError, "Metadata is missing key: description" unless meta.key?(:description)
+      if meta.key?(:cli_options)
+        fail InvalidMetadataError, "Metadata CLI options is not a hash" unless meta[:cli_options].is_a?(Hash)
+        meta[:cli_options].each_pair do |option, description|
+          fail InvalidMetadataError, "CLI option name is not a string" unless option.is_a?(String)
+          fail InvalidMetadataError, "CLI option details is not a string" unless description.is_a?(String)
+        end
+      end
     end
   end
 end

data/lib/aquatone/collectors/censys.rb

@@ -6,11 +6,14 @@ module Aquatone
         :author       => "James McLean (@vortexau)",
         :description  => "Uses the Censys API to find hostnames in TLS certificates",
         :require_keys => ["censys_secret","censys_id"],
+        :cli_options  => {
+          "censys-pages PAGES" => "Number of Censys API pages to process (default: 10)"
+        }
       }
 
-      API_BASE_URI         = "https://www.censys.io/api/v1".freeze
-      API_RESULTS_PER_PAGE = 100.freeze
-      PAGES_TO_PROCESS     = 10.freeze
+      API_BASE_URI             = "https://www.censys.io/api/v1".freeze
+      API_RESULTS_PER_PAGE     = 100.freeze
+      DEFAULT_PAGES_TO_PROCESS = 10.freeze
 
       def run
         request_censys_page
@@ -21,10 +24,10 @@ module Aquatone
 
           # Censys expects Basic Auth for requests.
           auth = {
-              :username => get_key('censys_id'), 
+              :username => get_key('censys_id'),
               :password => get_key('censys_secret')
           }
-   
+
           # Define this is JSON content
           headers = {
             'Content-Type' => 'application/json',
@@ -41,11 +44,11 @@ module Aquatone
 
           # Search API documented at https://censys.io/api/v1/docs/search
           response = post_request(
-              "#{API_BASE_URI}/search/certificates", 
+              "#{API_BASE_URI}/search/certificates",
               query.to_json,
               {
                   :basic_auth => auth,
-                  :headers => headers 
+                  :headers => headers
               }
           )
 
@@ -75,9 +78,15 @@ module Aquatone
       end
 
       def next_page?(page, body)
-          page <= PAGES_TO_PROCESS && body["metadata"]["pages"] && API_RESULTS_PER_PAGE * page < body["metadata"]["count"].to_i
+          page <= pages_to_process && body["metadata"]["pages"] && API_RESULTS_PER_PAGE * page < body["metadata"]["count"].to_i
       end
 
+      def pages_to_process
+        if has_cli_option?("censys-pages")
+          return get_cli_option("censys-pages").to_i
+        end
+        DEFAULT_PAGES_TO_PROCESS
+      end
     end
   end
 end

data/lib/aquatone/collectors/dictionary.rb

@@ -4,13 +4,25 @@ module Aquatone
       self.meta = {
         :name        => "Dictionary",
         :author      => "Michael Henriksen (@michenriksen)",
-        :description => "Uses a dictionary to find hostnames"
+        :description => "Uses a dictionary to find hostnames",
+        :cli_options => {
+          "wordlist WORDLIST" => "OPTIONAL: wordlist/dictionary file to use for subdomain bruteforcing"
+        }
       }
 
-      DICTIONARY = File.join(Aquatone::AQUATONE_ROOT, "subdomains.lst").freeze
+      DEFAULT_DICTIONARY = File.join(Aquatone::AQUATONE_ROOT, "subdomains.lst").freeze
 
       def run
-        dictionary = File.open(DICTIONARY, "r")
+        if has_cli_option?("wordlist")
+          file = File.expand_path(get_cli_option("wordlist"))
+          if !File.readable?(file)
+            failure("Wordlist file #{file} is not readable or does not exist")
+          end
+          dictionary = File.open(file, "r")
+        else
+          dictionary = File.open(DEFAULT_DICTIONARY, "r")
+        end
+
         dictionary.each_line do |subdomain|
           add_host("#{subdomain.strip}.#{domain.name}")
         end

data/lib/aquatone/collectors/gtr.rb

@@ -5,15 +5,18 @@ module Aquatone
         :name        => "Google Transparency Report",
         :author      => "Michael Henriksen (@michenriksen)",
         :description => "Uses Google Transparency Report to find hostnames",
-        :slug        => "gtr"
+        :slug        => "gtr",
+        :cli_options  => {
+          "gtr-pages PAGES" => "Number of Google Transparency Report pages to process (default: 30)"
+        }
       }
 
-      BASE_URI         = "https://www.google.com/transparencyreport/jsonp/ct/search"
-      PAGES_TO_PROCESS = 30.freeze
+      BASE_URI                 = "https://www.google.com/transparencyreport/jsonp/ct/search"
+      DEFAULT_PAGES_TO_PROCESS = 30.freeze
 
       def run
         token = nil
-        PAGES_TO_PROCESS.times do
+        pages_to_process.times do
           response = parse_response(request_page(token))
           response["results"].each do |result|
             host = result["subject"]
@@ -53,6 +56,13 @@ module Aquatone
         return false unless host.end_with?(".#{domain.name}")
         true
       end
+
+      def pages_to_process
+        if has_cli_option?("gtr-pages")
+          return get_cli_option("gtr-pages").to_i
+        end
+        DEFAULT_PAGES_TO_PROCESS
+      end
     end
   end
 end

data/lib/aquatone/collectors/netcraft.rb

@@ -4,18 +4,21 @@ module Aquatone
       self.meta = {
         :name        => "Netcraft",
         :author      => "Michael Henriksen (@michenriksen)",
-        :description => "Uses searchdns.netcraft.com to find hostnames"
+        :description => "Uses searchdns.netcraft.com to find hostnames",
+        :cli_options  => {
+          "netcraft-pages PAGES" => "Number of Netcraft pages to process (default: 10)"
+        }
       }
 
-      BASE_URI         = "http://searchdns.netcraft.com/".freeze
-      HOSTNAME_REGEX   = /<a href="http:\/\/(.*?)\/" rel="nofollow">/.freeze
-      RESULTS_PER_PAGE = 20.freeze
-      PAGES_TO_PROCESS = 10.freeze
+      BASE_URI                 = "http://searchdns.netcraft.com/".freeze
+      HOSTNAME_REGEX           = /<a href="http:\/\/(.*?)\/" rel="nofollow">/.freeze
+      RESULTS_PER_PAGE         = 20.freeze
+      DEFAULT_PAGES_TO_PROCESS = 10.freeze
 
       def run
         last  = nil
         count = 0
-        PAGES_TO_PROCESS.times do |i|
+        pages_to_process.times do |i|
           page = i + 1
           if page == 1
             uri = "#{BASE_URI}/?restriction=site+contains&host=*.#{url_escape(domain.name)}&lookup=wait..&position=limited"
@@ -43,6 +46,13 @@ module Aquatone
         end
         hosts
       end
+
+      def pages_to_process
+        if has_cli_option?("netcraft-pages")
+          return get_cli_option("netcraft-pages").to_i
+        end
+        DEFAULT_PAGES_TO_PROCESS
+      end
     end
   end
 end

data/lib/aquatone/collectors/shodan.rb

@@ -5,12 +5,15 @@ module Aquatone
         :name         => "Shodan",
         :author       => "Michael Henriksen (@michenriksen)",
         :description  => "Uses the Shodan API to find hostnames",
-        :require_keys => ["shodan"]
+        :require_keys => ["shodan"],
+        :cli_options  => {
+          "shodan-pages PAGES" => "Number of Shodan API pages to process (default: 10)"
+        }
       }
 
-      API_BASE_URI         = "https://api.shodan.io/shodan".freeze
-      API_RESULTS_PER_PAGE = 100.freeze
-      PAGES_TO_PROCESS     = 10.freeze
+      API_BASE_URI             = "https://api.shodan.io/shodan".freeze
+      API_RESULTS_PER_PAGE     = 100.freeze
+      DEFAULT_PAGES_TO_PROCESS = 10.freeze
 
       def run
         request_shodan_page
@@ -38,7 +41,14 @@ module Aquatone
       end
 
       def next_page?(page, body)
-        page <= PAGES_TO_PROCESS && body["total"] && API_RESULTS_PER_PAGE * page < body["total"].to_i
+        page <= pages_to_process && body["total"] && API_RESULTS_PER_PAGE * page < body["total"].to_i
+      end
+
+      def pages_to_process
+        if has_cli_option?("shodan-pages")
+          return get_cli_option("shodan-pages").to_i
+        end
+        DEFAULT_PAGES_TO_PROCESS
       end
     end
   end

data/lib/aquatone/commands/discover.rb

@@ -77,7 +77,7 @@ module Aquatone
           next if skip_collector?(collector)
           output("Running collector: #{bold(collector.meta[:name])}... ")
           begin
-            collector_instance = collector.new(@domain)
+            collector_instance = collector.new(@domain, options)
             hosts = collector_instance.execute!
             output("Done (#{hosts.count} #{hosts.count == 1 ? 'host' : 'hosts'})\n")
             @hosts += hosts

data/lib/aquatone/version.rb

@@ -1,3 +1,3 @@
 module Aquatone
-  VERSION = "0.3.0".freeze
+  VERSION = "0.4.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aquatone
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Michael Henriksen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-21 00:00:00.000000000 Z
+date: 2017-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty