aquatone 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d863c9389551c6e1e99644c5532a9765daba2d23
+  data.tar.gz: 3c21a3f09be1fce495168e11bc12bf93751e452a
+SHA512:
+  metadata.gz: dc2a6d6eb9c151dffedbff14ec9c6f8c7ad797a83824c07df0720e323fb294abd5f7618fb33bd0d18820f43aae07a017d2a63670449fca237840198a03410c05
+  data.tar.gz: 5ebb3c53736201578dabb6998bab3db1347f03f8cc1d48f7e6dbb6c7336c4b3a76a36708d1802b02a09560992ad7a9465876b5a0e831151bc40067fe7a84b147

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/node_modules/
+.ruby-version
+.ruby-gemset

data/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## [Unreleased]
+### Added
+
+### Changed
+
+## 0.1.0
+### Added
+- Initial release
+
+### Changed
+
+[Unreleased]: https://github.com/michenriksen/aquatone/compare/v0.1.0...HEAD

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aquatone.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Michael Henriksen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,168 @@
+# AQUATONE
+
+AQUATONE is a set of tools for performing reconnaissance on domain names. It can
+discover subdomains on a given domain by using open sources as well as the more
+common subdomain dictionary brute force approach. After subdomain discovery,
+AQUATONE can then scan the hosts for common web ports and HTTP headers, HTML
+bodies and screenshots can be gathered and consolidated into a report for easy
+analysis of the attack surface.
+
+## Installation
+
+### Dependencies
+
+AQUATONE depends on [Node.js] and [NPM] package manager for its web page screenshotting capabilities. Follow [this guide] for Installation instructions.
+
+You will also need a newer version of Ruby installed. If you plan to use AQUATONE in [Kali] Linux, you are already set up with this. If not, it is recommended to install Ruby with [RVM].
+
+Finally, the tool itself can be installed with the following command in a terminal:
+
+    $ gem install aquatone
+
+## Usage
+
+### Discovery
+
+The first stage of an AQUATONE assessment is the discovery stage where subdomains are discovered on the target domain using open sources, services and the more common dictionary brute force approach:
+
+    $ aquatone-discover --domain example.com
+
+aquatone-discover will find the target's nameservers and shuffle DNS lookups between them. Should a lookup fail on the target domain's nameservers, aquatone-discover will fall back to using Google's public DNS servers to maximize discovery. The fallback DNS servers can be changed with the `--fallback-nameservers` option:
+
+    $ aquatone-discover --domain example.com --fallback-nameservers 87.98.175.85,5.9.49.12
+
+#### Tuning
+
+aquatone-discover will use 5 threads as default for concurrently performing DNS lookups. This provides reasonable performance but can be tuned to be more or less aggressive with the `--threads` option:
+
+    $ aquatone-discover --domain example.com --threads 25
+
+Hammering a DNS server with failing lookups can potentially be picked up by intrusion detection systems, so if that is a concern for you, you can make aquatone-discover a bit more stealthy with the `--sleep` and `--jitter` options. `--sleep` accepts a number of seconds to sleep between each DNS lookup while `--jitter` accepts a percentage of the `--sleep` value to randomly add or subtract to or from the sleep interval in order to break the sleep pattern and make it less predictable.
+
+    $ aquatone-discover --domain example.com --sleep 5 --jitter 30
+
+Please note that setting the `--sleep` option will force the thread count to one. The `--jitter` option will only be considered if the `--sleep` option has also been set.
+
+#### API keys
+
+Some of the passive collectors will require API keys or similar credentials in order to work. Setting these values can be done with the `--set-key` option:
+
+    $ aquatone-discover --set-key shodan o1hyw8pv59vSVjrZU3Qaz6ZQqgM91ihQ
+
+All keys will be saved in  `~/aquatone/.keys.yml`.
+
+#### Results
+
+When aquatone-discover is finished, it will create a `hosts.txt` file in the `~/aquatone/<domain>` folder, so for a scan of example.com it would be located at `~/aquatone/example.com/hosts.txt`. The format will be a comma-separated list of hostnames and their IP, for example:
+
+    example.com,93.184.216.34
+    www.example.com,93.184.216.34
+    secret.example.com,93.184.216.36
+    cdn.example.com,192.0.2.42
+    ...
+
+In addition to the `hosts.txt` file, it will also generate a `hosts.json` which includes the same information but in JSON format. This format might be preferable if you want to use the information in custom scripts and tools. `hosts.json` will also be used by the aquatone-scan and aquatone-gather tools.
+
+See `aquatone-discover --help` for more options.
+
+### Scanning
+
+The scanning stage is where AQUATONE will enumerate the discovered hosts for open TCP ports that are commonly used for web services:
+
+    $ aquatone-scan --domain example.com
+
+The `--domain` option will look for `hosts.json` in the domain's AQUATONE assessment directory, so in the example above it would look for `~/aquatone/example.com/hosts.json`. This file should be present if `aquatone-discover --domain example.com` has been run previously.
+
+#### Ports
+
+By default, aquatone-scan will scan the following TCP ports: 80, 443, 8000, 8080 and 8443. These are very common ports for web services and will provide a reasonable coverage. Should you want to specifiy your own list of ports, you can use the `--ports` option:
+
+    $ aquatone-scan --domain example.com --ports 80,443,3000,8080
+
+Instead of a comma-separated list of ports, you can also specify one of the built-in list aliases:
+
+ * **small**: 80, 443
+ * **medium**: 80, 443, 8000, 8080, 8443 (same as default)
+ * **large**: 80, 81, 443, 591, 2082, 2095, 2096, 3000, 8000, 8001, 8008, 8080, 8083, 8443, 8834, 8888, 55672
+ * **huge**: 80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711, 4712, 4993, 5000, 5104, 5108, 5280, 5281, 5800, 6543, 7000, 7396, 7474, 8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8083, 8088, 8090, 8091, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333, 8337, 8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, 9090, 9091, 9200, 9443, 9800, 9981, 11371, 12443, 16080, 18091, 18092, 20720, 55672
+
+**Example:**
+
+    $ aquatone-scan --domain example.com --ports large
+
+#### Tuning
+
+Like aquatone-discover, you can make the scanning more or less aggressive with the `--threads` option which accepts a number of threads for concurrent port scans. The default number of threads is 5.
+
+    $ aquatone-scan --domain example.com --threads 25
+
+As aquatone-scan is performing port scanning, it can obviously be picked up by intrusion detection systems. While it will attempt to lessen the risk of detection by randomising hosts and ports, you can tune the stealthiness more with the `--sleep` and `--jitter` options which work just like the similarly named options for aquatone-discover. Keep in mind that setting the `--sleep` option will force the number of threads to one.
+
+#### Results
+
+When aquatone-scan is finished, it will create a `urls.txt` file in the `~/aquatone/<domain>` directory, so for a scan of example.com it would be located at `~/aquatone/example.com/urls.txt`. The format will be a list of URLs, for example:
+
+    http://example.com/
+    https://example.com/
+    http://www.example.com/
+    https://www.example.com/
+    http://secret.example.com:8001/
+    https://secret.example.com:8443/
+    http://cdn.example.com/
+    https://cdn.example.com/
+    ...
+
+This file can be loaded into other tools such as [EyeWitness].
+
+aquatone-scan will also generate a `open_ports.txt` file, which is a comma-separated list of hosts and their open ports, for example:
+
+    93.184.216.34,80,443
+    93.184.216.34,80
+    93.184.216.36,80,443,8443
+    192.0.2.42,80,8080
+    ...
+
+See `aquatone-scan --help` for more options.
+
+### Gathering
+
+The final stage is the gathering part where the results of the discovery and scanning stages are used to query the discovered web services in order to retrieve and save HTTP response headers and HTML bodies, as well as taking screenshots of how the web pages look like in a web browser to make analysis easier. The screenshotting is done with the [Nightmare.js] Node.js library. This library will be installed automatically if it's not present in the system.
+
+    $ aquatone-gather --domain example.com
+
+aquatone-gather will look for `hosts.json` and `open_ports.txt` in the given domain's AQUATONE assessment directory and request and screenshot every IP address for each domain name for maximum coverage.
+
+#### Tuning
+
+Like aquatone-discover and aquatone-scan, you can make the gathering more or less aggressive with the `--threads` option which accepts a number of threads for concurrent requests. The default number of threads is 5.
+
+    $ aquatone-gather --domain example.com --threads 25
+
+As aquatone-gather is interacting with web services, it can be picked up by intrusion detection systems. While it will attempt to lessen the risk of detection by randomising hosts and ports, you can tune the stealthiness more with the `--sleep` and `--jitter` options which work just like the similarly named options for aquatone-discover. Keep in mind that setting the `--sleep` option will force the number of threads to one.
+
+#### Results
+
+When aquatone-gather is finished, it will have created several directories in the domain's AQUATONE assessment directory:
+
+ * `headers/`: Contains text files with HTTP response headers from each web page
+ * `html/`: Contains text files with HTML response bodies from each web page
+ * `screenshots/`: Contains PNG images of how each web page looks like in a browser
+ * `report/` Contains report files in HTML displaying the gathered information for easy analysis
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/michenriksen/aquatone.
+
+
+## License
+
+AQUATONE is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+[Node.js]: https://nodejs.org/
+[NPM]: https://www.npmjs.com/
+[this guide]: https://docs.npmjs.com/getting-started/installing-node
+[Kali]: https://kali.org/
+[RVM]: https://rvm.io/
+[Nightmare.js]: http://www.nightmarejs.org/
+[EyeWitness]: https://www.christophertruncer.com/eyewitness-usage-guide/

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/aquatone.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aquatone/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "aquatone"
+  spec.version       = Aquatone::VERSION
+  spec.authors       = ["Michael Henriksen"]
+  spec.email         = ["michenriksen@neomailbox.ch"]
+
+  spec.summary       = %q{A tool for domain flyovers.}
+  spec.homepage      = "https://github.com/michenriksen/aquatone"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "httparty", "~> 0.14.0"
+  spec.add_dependency "childprocess", "~> 0.7.0"
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/aquatone.js

@@ -0,0 +1,164 @@
+var fs        = require('fs');
+var Nightmare = require('nightmare');
+var nightmare = Nightmare({
+  width: 1024,
+  height: 768,
+  switches: {
+    'ignore-certificate-errors': true
+  }
+});
+
+function debug(message) {
+  if (env["AQUATONE_DEBUG"]) {
+    console.log(message);
+  }
+}
+
+var USER_AGENTS = [
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0",
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.1 Safari/603.1.30",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0",
+  "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
+];
+
+var HTTP_RESPONSES = {
+  100: "Continue",
+  101: "Switching Protocols",
+  200: "OK",
+  201: "Created",
+  202: "Accepted",
+  203: "Non-Authoritative Information",
+  204: "No Content",
+  205: "Reset Content",
+  206: "Partial Content",
+  300: "Multiple Choices",
+  301: "Moved Permanently",
+  302: "Found",
+  303: "See Other",
+  304: "Not Modified",
+  305: "Use Proxy",
+  306: "(Unused)",
+  307: "Temporary Redirect",
+  400: "Bad Request",
+  401: "Unauthorized",
+  402: "Payment Required",
+  403: "Forbidden",
+  404: "Not Found",
+  405: "Method Not Allowed",
+  406: "Not Acceptable",
+  407: "Proxy Authentication Required",
+  408: "Request Timeout",
+  409: "Conflict",
+  410: "Gone",
+  411: "Length Required",
+  412: "Precondition Failed",
+  413: "Request Entity Too Large",
+  414: "Request-URI Too Long",
+  415: "Unsupported Media Type",
+  416: "Requested Range Not Satisfiable",
+  417: "Expectation Failed",
+  500: "Internal Server Error",
+  501: "Not Implemented",
+  502: "Bad Gateway",
+  503: "Service Unavailable",
+  504: "Gateway Timeout",
+  505: "HTTP Version Not Supported"
+};
+
+function randomIpAddress() {
+  var result = [];
+  for (var i = 0; i < 4; i++) {
+    result[i] = Math.floor(Math.random() * (255 - 1 + 1)) + 1;
+  }
+  return result.join(".");
+}
+
+function prettyHeader(header) {
+  var words = header.split("-");
+  for (var i = 0; i < words.length; i++) {
+    words[i] = words[i].charAt(0).toUpperCase() + words[i].slice(1);
+  }
+  return words.join("-");
+}
+
+var url                   = process.argv[2];
+var vhost                 = process.argv[3];
+var htmlDestination       = process.argv[4];
+var headersDestination    = process.argv[5];
+var screenshotDestination = process.argv[6];
+var visit = {
+  success: true,
+  url: url,
+  vhost: vhost
+};
+
+nightmare
+  .useragent(USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)])
+  .goto(url, {
+    'Host': vhost,
+    'Accept-Language': 'en-US,en;q=0.8',
+    'Accept-Encoding': '',
+    'Via': '1.1 ' + randomIpAddress(),
+    'X-Forwarded-For': randomIpAddress()
+  })
+  .then(function(result) {
+    visit.code   = result.code;
+    visit.status = result.code + " " + (HTTP_RESPONSES[visit.code] || "(unknown)")
+    var headers  = {};
+    for (var k in result.headers) {
+      headers[prettyHeader(k)] = result.headers[k].join(" ");
+    }
+    visit.headers = headers;
+    return nightmare
+      .wait(1000)
+      .evaluate(function() {
+        document.body.bgColor = 'white';
+      })
+      .html(htmlDestination, 'HTMLOnly')
+      .then(function(result) {
+        return nightmare
+          .screenshot(screenshotDestination, {
+            x: 0,
+            y: 0,
+            width: 1024,
+            height: 768
+          })
+          .end()
+          .then(function() {
+            headers = [];
+            for (var k in visit.headers) {
+              headers.push(k + ": " + visit.headers[k]);
+            }
+            fs.writeFile(headersDestination, headers.join("\r\n") + "\r\n", function(err) {
+              console.log(JSON.stringify(visit));
+              process.exit(0);
+            });
+          })
+      })
+  })
+  .catch(function(error) {
+    console.log(JSON.stringify({
+      success: false,
+      url: url,
+      vhost: vhost,
+      error: error.message,
+      code: error.code,
+      details: error.details
+    }));
+    process.exit(1);
+  });