aquatone 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78769d828a09c8c64d631762851816956390be05
-  data.tar.gz: fed735f12ef3d389c87033f55b7ccf2570afc985
+  metadata.gz: 1572ba4c07838fd7294dfc001c8ff23b2f7c19fc
+  data.tar.gz: 2c7fc94f9cef66e71fc8adeed1840a3170cf6367
 SHA512:
-  metadata.gz: d1a29f270be14944375aa6b9d0231a81a37484d8b4ad5a9561c354cc4e7e40fe0385e1998dd4fee7cc6985741d04d867520d8be03f0e8c32ce8c5bba79e50980
-  data.tar.gz: 2738fb5ced2b632399b26d42ed25d9c2e7cbbc84d2ee398cf1c111264fda511e4d696c725e64e21b2dbc8256058afc9f7486f05fa5499f8cca8bbf9362f0d047
+  metadata.gz: d5086248f07db25eeead317a570ee56cf3d622eb72e471d85c1c33611f3cfc4bba0be721a9f65658e6e078b264bbbb2327157574070e235d1542c4a85ba4ec6e
+  data.tar.gz: f81b9ab7409889e524c59fda6e53e7ce4e891a515f7492885f5fcd5e558ae8bdb187c2e6295ad5ee40324f51b12994dfbc39759a272a03fbcbd08b75e5f90fc6

data/CHANGELOG.md

@@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
+
+## [0.2.0]
+### Added
+ - New Collector: riddler.io (Thanks, [@jolle](https://github.com/jolle)!)
+ - New Collector: crt.sh (Thanks, [@jolle](https://github.com/jolle)!)
+ - New Collector: censys.io (Thanks, [@vortexau](https://github.com/vortexau)!)
+ - New Collector: passivetotal.org
+
+### Changed
+ - Capture potential `NameError` exception in `asked_for_progress?` method,
+   related to known JRuby bug (issue #4)
+ - Capture potential `Errno::EBADF` exception in `asked_for_progress?` method (issue #15)
+ - Improve handling of error when aquatone-gather is run on a system without a graphical desktop session (X11)
+ - Exclude hosts resolving to broadcast addresses in aquatone-discover (issue #11)
+
+
 ## [0.1.1]
 ### Added
 
@@ -22,5 +38,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
-[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.1.0...HEAD
-[0.1.1]: https://github.com/olivierlacan/keep-a-changelog/compare/v0.1.0...v0.1.1
+[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/michenriksen/aquatone/compare/v0.1.1...v0.2.0
+[0.1.1]: https://github.com/michenriksen/aquatone/compare/v0.1.0...v0.1.1

data/README.md

@@ -19,6 +19,9 @@ Finally, the tool itself can be installed with the following command in a termin
 
     $ gem install aquatone
 
+**IMPORTANT:** AQUATONE's screenshotting capabilities depend on being run on a system with a graphical desktop environment. It is strongly recommended to install and run AQUATONE in a [Kali linux](https://www.kali.org/) virtual machine.
+**I will not provide support or bug fixing for other systems than Kali Linux.**
+
 ## Usage
 
 ### Discovery

data/lib/aquatone/browser/drivers/nightmare.rb

@@ -1,6 +1,9 @@
 module Aquatone
   class Browser
     module Drivers
+
+      class IncompatabilityError < StandardError; end
+
       class Nightmare
         attr_reader :url, :vhost, :html_destination, :headers_destination, :screenshot_destination, :options
 
@@ -23,6 +26,8 @@ module Aquatone
           wout.close
           command_output = rout.readlines.join("\n").strip
           JSON.parse(command_output)
+        rescue JSON::ParserError
+          fail IncompatabilityError, "Nightmarejs must be run on a system with a graphical desktop session (X11)"
         rescue => e
           process.stop if process
           return {

data/lib/aquatone/collector.rb

@@ -56,7 +56,7 @@ module Aquatone
     end
 
     def get_request(uri, options={})
-      Aquatone::HttpClient.get(uri)
+      Aquatone::HttpClient.get(uri, options)
     end
 
     def post_request(uri, body=nil, options={})

data/lib/aquatone/collectors/censys.rb

@@ -0,0 +1,83 @@
+module Aquatone
+  module Collectors
+    class Censys < Aquatone::Collector
+      self.meta = {
+        :name         => "Censys",
+        :author       => "James McLean (@vortexau)",
+        :description  => "Uses the Censys API to find hostnames in TLS certificates",
+        :require_keys => ["censys_secret","censys_id"],
+      }
+
+      API_BASE_URI         = "https://www.censys.io/api/v1".freeze
+      API_RESULTS_PER_PAGE = 100.freeze
+      PAGES_TO_PROCESS     = 10.freeze
+
+      def run
+        request_censys_page
+      end
+
+      def request_censys_page(page=1)
+          # Initial version only supporting Censys Certificates API
+
+          # Censys expects Basic Auth for requests.
+          auth = {
+              :username => get_key('censys_id'), 
+              :password => get_key('censys_secret')
+          }
+   
+          # Define this is JSON content
+          headers = {
+            'Content-Type' => 'application/json',
+            'Accept' => 'application/json'
+          }
+
+          # The post body itself, as JSON
+          query = {
+              'query'   => url_escape("#{domain.name}"),
+              'page'    => page,
+              'fields'  => [ "parsed.names", "parsed.extensions.subject_alt_name.dns_names" ],
+              'flatten' => true
+          }
+
+          # Search API documented at https://censys.io/api/v1/docs/search
+          response = post_request(
+              "#{API_BASE_URI}/search/certificates", 
+              query.to_json,
+              {
+                  :basic_auth => auth,
+                  :headers => headers 
+              }
+          )
+
+          if response.code != 200
+              failure(response.parsed_response["error"] || "Censys API encountered error: #{response.code}")
+          end
+
+          # If nothing returned from Censys, return:
+          return unless response.parsed_response["results"]
+
+          response.parsed_response["results"].each do |result|
+
+            next unless result["parsed.extensions.subject_alt_name.dns_names"]
+            result["parsed.extensions.subject_alt_name.dns_names"].each do |altdns|
+                add_host(altdns) if altdns.end_with?(".#{domain.name}")
+            end
+
+            next unless result["parsed.names"]
+            result["parsed.names"].each do |parsedname|
+                add_host(parsedname) if parsedname.end_with?(".#{domain.name}")
+            end
+          end
+
+          # Get the next page of results
+          request_censys_page(page + 1) if next_page?(page, response.parsed_response)
+
+      end
+
+      def next_page?(page, body)
+          page <= PAGES_TO_PROCESS && body["metadata"]["pages"] && API_RESULTS_PER_PAGE * page < body["metadata"]["count"].to_i
+      end
+
+    end
+  end
+end

data/lib/aquatone/collectors/crtsh.rb

@@ -0,0 +1,19 @@
+module Aquatone
+  module Collectors
+    class Crtsh < Aquatone::Collector
+      self.meta = {
+        :name         => "Certificate Search",
+        :author       => "Joel (@jolle)",
+        :description  => "Uses crt.sh by COMODO CA to find hostnames"
+      }
+
+      def run
+        response = get_request("https://crt.sh/?dNSName=%25.#{url_escape(domain.name)}")
+
+        response.body.to_enum(:scan, /<TD>([a-zA-Z0-9\*_.-]+\.#{Regexp.escape(domain.name)})<\/TD>/).map do |column|
+          add_host(column[0].gsub("*.", ""))
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/passivetotal.rb

@@ -0,0 +1,31 @@
+module Aquatone
+  module Collectors
+    class Passivetotal < Aquatone::Collector
+      self.meta = {
+        :name         => "PassiveTotal",
+        :author       => "Michael Henriksen (@michenriksen)",
+        :description  => "Uses RiskIQ's PassiveTotal API to find hostnames",
+        :require_keys => ["passivetotal_key", "passivetotal_secret"]
+      }
+
+      API_BASE_URI = "https://api.passivetotal.org".freeze
+
+      def run
+        response = get_request(
+          "#{API_BASE_URI}/v2/enrichment/subdomains?query=.#{url_escape(domain.name)}",
+          :basic_auth => {:username => get_key("passivetotal_key"), :password => get_key("passivetotal_secret")}
+        )
+        body = response.parsed_response
+        if response.code != 200
+          failure(failure(body["message"] || "PassiveTotal API returned unexpected response code: #{response.code}"))
+        end
+        if !(body.key?("success") && body["success"])
+          failure("Request failed")
+        end
+        body["subdomains"].each do |subdomain|
+          add_host("#{subdomain}.#{domain.name}")
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/riddler.rb

@@ -0,0 +1,43 @@
+module Aquatone
+  module Collectors
+    class Riddler < Aquatone::Collector
+      self.meta = {
+        :name         => "Riddler",
+        :author       => "Joel (@jolle)",
+        :description  => "Uses Riddler by F-Secure to find hostnames",
+        :require_keys => ["riddler_username", "riddler_password"]
+      }
+
+      API_BASE_URI = "https://riddler.io"
+
+      def run
+        auth_response = post_request("#{API_BASE_URI}/auth/login", {
+          :email    => get_key("riddler_username"),
+          :password => get_key("riddler_password")
+        }.to_json, {
+          :headers  => { "Content-Type" => "application/json" }
+        })
+
+        if auth_response.code == 400 or auth_response.parsed_response["meta"]["code"] == 400
+          failure("Invalid credentials to Riddler.io")
+        elsif auth_response.code != 200 or auth_response.parsed_response["meta"]["code"] != 200
+          failure("Riddler.io auth API returned unexpected response code: #{response.code}")
+        end
+        
+        token = auth_response.parsed_response["response"]["user"]["authentication_token"]
+
+        response = post_request("#{API_BASE_URI}/api/search", {
+          :query  => "pld:#{url_escape(domain.name)}",
+          :output => "host",
+          :limit  => 500
+        }.to_json, {
+          :headers  => { "Content-Type" => "application/json", "Authentication-Token" => token }
+        })
+
+        response.parsed_response.each do |record|
+          add_host(record["host"]) unless record["host"].empty?
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/command.rb

@@ -112,14 +112,18 @@ module Aquatone
     end
 
     def asked_for_progress?
-      while c = STDIN.read_nonblock(1)
-        return true if c == "\n"
+      begin
+        while c = STDIN.read_nonblock(1)
+          return true if c == "\n"
+        end
+        false
+      rescue IO::EAGAINWaitReadable, Errno::EBADF
+        false
+      rescue Errno::EINTR, Errno::EAGAIN, EOFError
+        true
       end
+    rescue NameError
       false
-    rescue IO::EAGAINWaitReadable
-      false
-    rescue Errno::EINTR, Errno::EAGAIN, EOFError
-      true
     end
 
     def has_executable?(name)

data/lib/aquatone/commands/discover.rb

@@ -103,8 +103,7 @@ module Aquatone
                    "#{(task_count.to_f / @hosts.count.to_f * 100.00).round(1)}% done\n")
           end
           if ip = @resolver.resolve(host)
-            next if wildcard_ip?(ip)
-            next if (options[:ignore_private] && private_ip?(ip))
+            next if exclude_ip?(ip)
             @host_dictionary[host] = ip
             output("#{ip.ljust(15)} #{bold(host)}\n")
           end
@@ -157,6 +156,10 @@ module Aquatone
            0 1 2 3 4 5 6 7 8 9).shuffle.take(10).join
       end
 
+      def exclude_ip?(ip)
+        wildcard_ip?(ip) || (options[:ignore_private] && private_ip?(ip)) || broadcast_ip?(ip)
+      end
+
       def wildcard_ip?(ip)
         @wildcard_ips.include?(ip)
       end
@@ -165,6 +168,10 @@ module Aquatone
         ip =~ /(\A127\.)|(\A10\.)|(\A172\.1[6-9]\.)|(\A172\.2[0-9]\.)|(\A172\.3[0-1]\.)|(\A192\.168\.)/
       end
 
+      def broadcast_ip?(ip)
+        ip == "255.255.255.255"
+      end
+
       def skip_collector?(collector)
         if options[:only_collectors]
           if options[:only_collectors].include?(collector.sluggified_name)

data/lib/aquatone/commands/gather.rb

@@ -78,17 +78,23 @@ module Aquatone
         @tasks.shuffle.each do |task|
           host, port, domain = task
           pool.schedule do
-            output_progress if asked_for_progress?
-            visit = visit_page(host, port, domain)
-            if visit['success']
-              output("#{green('Processed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['status']}\n")
-              @successful += 1
-            else
-              output("   #{red('Failed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['error']} #{visit['details']}\n")
-              @failed += 1
+            begin
+              output_progress if asked_for_progress?
+              visit = visit_page(host, port, domain)
+              if visit['success']
+                output("#{green('Processed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['status']}\n")
+                @successful += 1
+              else
+                output("   #{red('Failed:')} #{Aquatone::UrlMaker.make(host, port)} (#{domain}) - #{visit['error']} #{visit['details']}\n")
+                @failed += 1
+              end
+              jitter_sleep
+              @task_count += 1
+            rescue Aquatone::Browser::Drivers::IncompatabilityError => e
+              output("\n")
+              output(red("Incompatability Error:") + " #{e.message}\n")
+              exit 1
             end
-            jitter_sleep
-            @task_count += 1
           end
         end
         pool.shutdown

data/lib/aquatone/version.rb

@@ -1,3 +1,3 @@
 module Aquatone
-  VERSION = "0.1.1".freeze
+  VERSION = "0.2.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aquatone
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Henriksen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-18 00:00:00.000000000 Z
+date: 2017-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -108,11 +108,15 @@ files:
 - lib/aquatone/browser.rb
 - lib/aquatone/browser/drivers/nightmare.rb
 - lib/aquatone/collector.rb
+- lib/aquatone/collectors/censys.rb
+- lib/aquatone/collectors/crtsh.rb
 - lib/aquatone/collectors/dictionary.rb
 - lib/aquatone/collectors/dnsdb.rb
 - lib/aquatone/collectors/gtr.rb
 - lib/aquatone/collectors/hackertarget.rb
 - lib/aquatone/collectors/netcraft.rb
+- lib/aquatone/collectors/passivetotal.rb
+- lib/aquatone/collectors/riddler.rb
 - lib/aquatone/collectors/shodan.rb
 - lib/aquatone/collectors/threatcrowd.rb
 - lib/aquatone/collectors/virustotal.rb