aquatone 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: baf48eeafa767353c69d4a516f55acf3a8564fe1
-  data.tar.gz: b5abde13dde9cf2cda34c4dca4f8d69184b0b56d
+  metadata.gz: c14050e82d9cc20270db7e87a40e6bcc0a1e3e50
+  data.tar.gz: 9a3c750f6c31b0fa4d04484fa945b88a9b1a42fa
 SHA512:
-  metadata.gz: d174b9c82b96992bdc75a8f4ffb9e9d74c93a901ec58478487a28c597d769382a1e7b68cc5bd79a7d0bcc0e54175c74be6e61539f56d06e3d848ef002d52d270
-  data.tar.gz: 062654c4512f11b8abf86b621b47dd6183ffa096eb2bb54b06cfe3fe4c3985a74d0aff9314f64583cc0f83f3314c24cf52ccdc2ac02bb459030ce4f1949b7aaf
+  metadata.gz: 3450460b07ac57523d99708dbce207a2ce408996099253a30985a2c401ba961e1ed46ccdf217a78740f44f95771dfcbf5a0363a04dc1080bcbf4177b9243e60d
+  data.tar.gz: bce8b495f0e4f190ee123e89ddbe70addf9e0ebc73bf48330a604493cb6b0a70b9dcfeb797dc181edd1ab11d1ebdbcf628e299d87d24c43e2c706ffad19a4b77

data/CHANGELOG.md

@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
+## [0.5.0]
+
+### Added
+ - New Collector: Wayback Machine (archive.org) (Thanks, [@jolle](https://github.com/jolle)!)
+ - New Collector: ptrarchive.com
+ - New Collector: publicwww.com
+
+### Changed
+
+### Fixed
+ - Detect false positive situation in fastly.com Takeover module (Thanks, [@ramimac](https://github.com/ramimac)!)
+
 ## [0.4.1]
 
 ### Added
@@ -68,9 +80,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
-[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.4.1...HEAD
+[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.5.0...HEAD
+[0.5.0]: https://github.com/michenriksen/aquatone/compare/v0.4.1...v0.5.0
 [0.4.1]: https://github.com/michenriksen/aquatone/compare/v0.4.0...v0.4.1
 [0.4.0]: https://github.com/michenriksen/aquatone/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/michenriksen/aquatone/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/michenriksen/aquatone/compare/v0.1.1...v0.2.0
 [0.1.1]: https://github.com/michenriksen/aquatone/compare/v0.1.0...v0.1.1
+

data/lib/aquatone/collector.rb

@@ -78,6 +78,10 @@ module Aquatone
       CGI.escape(string)
     end
 
+    def regex_escape(string)
+      Regexp.escape(string)
+    end
+
     def random_sleep(seconds)
       random_sleep = ((1 - (rand(30) * 0.01)) * seconds.to_i)
       sleep(random_sleep)

data/lib/aquatone/collectors/ptrarchive.rb

@@ -0,0 +1,21 @@
+module Aquatone
+  module Collectors
+    class Ptrarchive < Aquatone::Collector
+      self.meta = {
+        :name        => "PTRArchive",
+        :author      => "Michael Henriksen (@michenriksen)",
+        :description => "Uses ptrarchive.com to find subdomains"
+      }
+
+      def run
+        response = get_request("http://ptrarchive.com/tools/search.htm?label=#{url_escape(domain.name)}&date=ALL")
+        if response.code != 200
+          failure("PTRArchive returned unexpected response code: #{response.code}")
+        end
+        response.body.scan(/[a-z0-9\.\-_]+\.#{regex_escape(domain.name)}/).each do |host|
+          add_host(host)
+        end
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/publicwww.rb

@@ -0,0 +1,34 @@
+module Aquatone
+  module Collectors
+    class Publicwww < Aquatone::Collector
+      self.meta = {
+        :name        => "PublicWWW",
+        :author      => "Michael Henriksen (@michenriksen)",
+        :description => "Uses the publicwww.com source code search engine to find subdomains",
+        :cli_options => {
+          "publicwww-pages PAGES" => "Number of PublicWWW pages to process (default: 30)"
+        }
+      }
+
+      DEFAULT_PAGES_TO_PROCESS = 30.freeze
+
+      def run
+        pages_to_process.times do |page|
+          response = get_request("https://publicwww.com/websites/.#{url_escape(domain.name)}/#{page + 1}")
+          response.body.gsub("<b>", "").gsub("</b>", "").scan(/[a-z0-9\.\-_]+\.#{regex_escape(domain.name)}/).each do |host|
+            add_host(host)
+          end
+        end
+      end
+
+      private
+
+      def pages_to_process
+        if has_cli_option?("publicwww-pages")
+          return get_cli_option("publicwww-pages").to_i
+        end
+        DEFAULT_PAGES_TO_PROCESS
+      end
+    end
+  end
+end

data/lib/aquatone/collectors/wayback_machine.rb

@@ -0,0 +1,41 @@
+require 'uri'
+
+module Aquatone
+  module Collectors
+    class WaybackMachine < Aquatone::Collector
+      self.meta = {
+        :name         => "Wayback Machine",
+        :author       => "Joel (@jolle)",
+        :description  => "Uses Wayback Machine by Internet Archive to find unique hostnames",
+        :cli_options  => {
+          "wayback-machine-timeout SECONDS" => "Timeout for Wayback Machine collector in seconds (default: 30)"
+        }
+      }
+
+      DEFAULT_TIMEOUT = 30.freeze
+
+      def run
+        response = nil
+        Timeout::timeout(timeout) do
+          response = get_request("http://web.archive.org/cdx/search/cdx?url=*.#{url_escape(domain.name)}&output=json&fl=original&collapse=urlkey")
+        end
+        response.parsed_response.each do |page|
+          if page[0] != "original"
+            begin
+              add_host(URI.parse(page[0]).host)
+            rescue URI::Error; end
+          end
+        end
+      end
+
+      private
+
+      def timeout
+        if has_cli_option?("wayback-machine-timeout")
+          return get_cli_option("wayback-machine-timeout").to_i
+        end
+        DEFAULT_TIMEOUT
+      end
+    end
+  end
+end

data/lib/aquatone/commands/discover.rb

@@ -84,6 +84,8 @@ module Aquatone
           rescue Aquatone::Collector::MissingKeyRequirement => e
             output(yellow("Skipped\n"))
             output(yellow(" -> #{e.message}\n"))
+          rescue Timeout::Error
+            output(red("Timed out\n"))
           rescue => e
             output(red("Error\n"))
             output(red(" -> #{e.message}\n"))

data/lib/aquatone/detectors/fastly.rb

@@ -9,12 +9,15 @@ module Aquatone
       }
 
       CNAME_VALUE          = ".fastly.net".freeze
+      CNAME_FALSE_POSITIVE = ".map.fastly.net".freeze
       RESPONSE_FINGERPRINT = "Fastly error: unknown domain".freeze
 
       def run
         return false unless cname_resource?
         if resource_value.end_with?(CNAME_VALUE)
-          return get_request("http://#{host}/").body.include?(RESPONSE_FINGERPRINT)
+          unless resource_value.end_with?(CNAME_FALSE_POSITIVE)
+            return get_request("http://#{host}/").body.include?(RESPONSE_FINGERPRINT)
+          end
         end
         false
       end

data/lib/aquatone/version.rb

@@ -1,3 +1,3 @@
 module Aquatone
-  VERSION = "0.4.1".freeze
+  VERSION = "0.5.0".freeze
 end

data/subdomains.lst

@@ -3343,6 +3343,7 @@ jedi
 jeff
 jemmy
 jenkins
+jenkins101
 jenkins-01
 jenkins-02
 jenkins-03

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aquatone
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Michael Henriksen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-22 00:00:00.000000000 Z
+date: 2017-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -118,10 +118,13 @@ files:
 - lib/aquatone/collectors/hackertarget.rb
 - lib/aquatone/collectors/netcraft.rb
 - lib/aquatone/collectors/passivetotal.rb
+- lib/aquatone/collectors/ptrarchive.rb
+- lib/aquatone/collectors/publicwww.rb
 - lib/aquatone/collectors/riddler.rb
 - lib/aquatone/collectors/shodan.rb
 - lib/aquatone/collectors/threatcrowd.rb
 - lib/aquatone/collectors/virustotal.rb
+- lib/aquatone/collectors/wayback_machine.rb
 - lib/aquatone/command.rb
 - lib/aquatone/commands/discover.rb
 - lib/aquatone/commands/gather.rb