aquanaut 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4173052655cbf46e9072ff1d04cfd6611f33f88b
+  data.tar.gz: 10a5195c0f2c45d6178a8eac71737247d88449e6
+SHA512:
+  metadata.gz: f6b6e7ac135045f018e9c6ab3ac8df090dc3ad7db6840412196bcccb284f82147b0a3257bbe3aa596c66cde770b0df34d00ce7b2dac66c9e01a9dca48acaf9f0
+  data.tar.gz: 1d4e01f0b10794e2d10be888d0e6aba0e8d2d7563369603bf637928fa51af6b0183c7a976967f51aaa73fb56a408eab77f942beebeea78e602f1d96925dfb5cf

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+
+rvm:
+  - 2.1.0
+
+script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,20 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aquanaut.gemspec
+gemspec
+
+gem 'public_suffix'
+gem 'mechanize'
+gem 'slim'
+gem 'webmock'
+
+group :test do
+  gem 'rspec'
+  gem 'rspec-core'
+  gem 'guard-rspec', require: false
+end
+
+group :development, :test do
+  gem 'pry'
+  gem 'pry-byebug'
+end

data/Guardfile

@@ -0,0 +1,24 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml|\.slim)$})          { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+
+  # Capybara features specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml|slim)$})     { |m| "spec/features/#{m[1]}_spec.rb" }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Konrad Reiche
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Aquanaut
+
+[![Build Status](https://travis-ci.org/platzhirsch/aquanaut.png)](http://travis-ci.org/platzhirsch/aquanaut)
+
+A web crawler that stays on a given domain and creates a graph representing the different pages, static assets and how they are interlinked.
+
+<img src="http://konrad-reiche.com/images/aquanaut.png">
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'aquanaut'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install aquanaut
+
+## Usage
+
+Execute `aquanaut` and specify the domain on which it should be executed.
+
+    $ aquanaut 'http://www.konrad-reiche.com'
+
+The results are written into the directory `sitemap`.
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/aquanaut/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/aquanaut.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aquanaut/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "aquanaut"
+  spec.version       = Aquanaut::VERSION
+  spec.authors       = ["Konrad Reiche"]
+  spec.email         = ["konrad.reiche@gmail.com"]
+  spec.summary       = %q{Aquanaut creates a sitemap dynamically based on a given domain.}
+  spec.description   = %q{Aquanaut creates a sitemap dynamically based on a given domain.}
+  spec.homepage      = "https://github.com/platzhirsch/aquanaut"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake", "~> 0.9"
+  spec.add_development_dependency "public_suffix", "~> 1.4", ">= 1.4.0"
+  spec.add_development_dependency "mechanize", "~> 2.7", ">= 2.7.3"
+  spec.add_development_dependency "slim", "~> 2.0", ">= 2.0.1"
+  spec.add_development_dependency "webmock", "~> 1.15", ">= 1.15.2"
+end

data/bin/aquanaut

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'aquanaut'
+
+if ARGV.empty?
+  raise ArgumentError, "Specify a target domain in the first argument"
+end
+
+target_domain = ARGV[0]
+
+if target_domain =~ URI::regexp
+  graph = Aquanaut.process_domain(target_domain)
+  Aquanaut::Sitemap.new(graph, target_domain).render_results
+else
+  raise ArgumentError, "#{target_domain} is not a valid URI"
+end
+

data/lib/aquanaut/asset_node.rb

@@ -0,0 +1,23 @@
+require 'aquanaut/page_node'
+
+module Aquanaut
+
+  # An asset node is a node that represents a static asset. The type specifies
+  # what kind of static asset it is, for instance image or stylesheet.
+  class AssetNode < PageNode
+
+    attr_reader :type
+
+    # Constructor
+    #
+    # @param [URI] uri identifying the static asset uniquely.
+    #
+    # @param [String] type specifying the kind of static asset.
+    #
+    def initialize(uri, type)
+      @type = type
+      super(uri)
+    end
+
+  end
+end

data/lib/aquanaut/graph.rb

@@ -0,0 +1,74 @@
+require 'json'
+
+# A graph representing the sitemap in terms of a data structure. A hash is
+# used internally to make the nodes accessible through the URIs.
+# 
+class Aquanaut::Graph
+  include Enumerable
+
+  def initialize
+    @nodes = Hash.new
+  end
+
+  # Use this method for making nodes available in the graph. New nodes are
+  # only assigned once.
+  #
+  # @param [Node] node the node to add to the graph.
+  # 
+  def add_node(node)
+    @nodes[node.uri] ||= node
+  end
+
+  # Use this method to easily add new edges without the need to pass actual
+  # node objects. The method delegates the edge creation to the dedicated node
+  # edge method.
+  #
+  # @param [URI] predecessor_uri source node for the edge
+  # @param [URI] successor_uri target node for the edge
+  #
+  def add_edge(predecessor_uri, successor_uri)
+    @nodes[predecessor_uri].add_edge(@nodes[successor_uri])
+  end
+
+  # Accessor method to retrieve nodes by their URI.
+  #
+  # @param [URI] uri the URI representing the node.
+  #
+  def [](uri)
+    @nodes[uri]
+  end
+
+  # Accessor method to iterate the nodes and their adjacency list.
+  #
+  def each
+    @nodes.values.each do |node|
+      yield node, node.adjacency_list
+    end
+  end
+
+  # Used for visualizing the graph on the front-end.
+  #
+  def to_json
+    model = { 'nodes' => [], 'links' => [] }
+
+    self.each do |node, adjacency|
+      if node.instance_of?(Aquanaut::PageNode)
+        group = 1
+      else
+        asset_groups = { 'image' => 2, 'stylesheet' => 3 }
+        group = asset_groups[node.type]
+      end
+
+      model['nodes'] << { 'name' => node.uri, 'group' => group }
+      source = @nodes.values.index(node)
+
+      adjacency.each do |adjacency_node|
+        target = @nodes.values.index(adjacency_node)
+        model['links'] << { 'source' => source, 'target' => target }
+      end
+    end
+
+    return model.to_json
+  end
+
+end

data/lib/aquanaut/node.rb

@@ -0,0 +1,22 @@
+module Aquanaut
+
+  # Base node class which needs to be inherited for special cases.
+  #
+  # @abstract
+  #
+  class Node
+
+    attr_reader :adjacency_list
+
+    def initialize()
+      @adjacency_list = []
+    end
+
+    # Implements adjacency with an adjacency list.
+    #
+    def add_edge(successor)
+      @adjacency_list << successor
+    end
+
+  end
+end

data/lib/aquanaut/page_node.rb

@@ -0,0 +1,25 @@
+require 'aquanaut/node'
+
+module Aquanaut
+
+  # A page node represents an actual page in the specified domain.
+  #
+  class PageNode < Node
+
+    attr_reader :uri
+
+    def initialize(uri)
+      @uri = uri
+      super()
+    end
+
+    # Display method used on the front-end for the sitemap in list format.
+    #
+    def display
+      part = "#{@uri.path}#{@uri.query}#{@uri.fragment}"
+      part = @uri.to_s if part.empty?
+      return part
+    end
+
+  end
+end

data/lib/aquanaut/sitemap.rb

@@ -0,0 +1,55 @@
+require 'pathname'
+require 'slim'
+
+# The sitemap class is used to render the results in HTML and JavaScript.
+# 
+# Uses SLIM as a template engine.
+#
+class Aquanaut::Sitemap
+
+  def initialize(graph, domain, target_dir="#{Dir.pwd}/sitemap")
+    @graph = graph
+    @domain = domain
+    @target_dir = target_dir
+
+    if Pathname.new(target_dir).relative?
+      @target_dir = File.expand_path("../../../#{target_dir}", __FILE__)
+    end
+  end
+
+  # Renders the results by initiailizing the dependencies and processingt the template.
+  #
+  def render_results
+    initialize_target_directory
+
+    options = { disable_escape: true }
+    template_path = File.expand_path('../templates/index.html.slim', __FILE__)
+    rendered_template = Slim::Template.new(template_path, options).render(self)
+
+    File.open("#{@target_dir}/index.html", 'w') do |file|
+      file.write rendered_template
+    end
+  end
+
+  private
+
+  # There are several asset files required. Vendor asset files, but also local
+  # asset files. They need to the copied to the target directory in order to
+  # work properly.
+  #
+  # @private
+  #
+  def initialize_target_directory
+    # create result directory
+    Dir.mkdir(@target_dir) unless Dir.exists?(@target_dir)
+
+    # copy vendor assets
+    vendor_dir = File.expand_path('../../../vendor/assets', __FILE__)
+    FileUtils.cp_r(vendor_dir, @target_dir, remove_destination: true)
+
+    # copy local assets
+    assets_dir = File.expand_path('../templates/assets', __FILE__)
+    FileUtils.cp_r(assets_dir, @target_dir)
+  end
+
+end

data/lib/aquanaut/templates/assets/css/custom.css

@@ -0,0 +1,27 @@
+h1 {
+  margin-bottom: 2em;
+  font-size: 28px;
+}
+
+h2 {
+  margin-bottom: 0.5em;
+  font-size: 24px;
+}
+
+#graph {
+  width: 960px;
+  height: 500px;
+  border: 1px solid #ccc;
+  margin-bottom: 2em;
+}
+
+.node {
+  stroke: #fff;
+  stroke-width: 1.5px;
+}
+
+.link {
+  stroke: #999;
+  stroke-opacity: .6;
+  stroke-width: 1px;
+}

data/lib/aquanaut/templates/assets/js/graph.js

@@ -0,0 +1,46 @@
+$(document).ready(function() {
+  var width = 960,
+      height = 500;
+
+  var color = d3.scale.category10();
+
+  var force = d3.layout.force()
+      .charge(-120)
+      .linkDistance(30)
+      .size([width, height]);
+
+  var svg = d3.select("#graph").append("svg")
+      .attr("width", width)
+      .attr("height", height);
+
+  force
+      .nodes(graph.nodes)
+      .links(graph.links)
+      .start();
+
+  var link = svg.selectAll(".link")
+      .data(graph.links)
+    .enter().append("line")
+      .attr("class", "link");
+
+  var node = svg.selectAll(".node")
+      .data(graph.nodes)
+    .enter().append("circle")
+      .attr("class", "node")
+      .attr("r", 5)
+      .style("fill", function(d) { return color(d.group); })
+      .call(force.drag);
+
+  node.append("title")
+      .text(function(d) { return d.name; });
+
+  force.on("tick", function() {
+    link.attr("x1", function(d) { return d.source.x; })
+        .attr("y1", function(d) { return d.source.y; })
+        .attr("x2", function(d) { return d.target.x; })
+        .attr("y2", function(d) { return d.target.y; });
+
+    node.attr("cx", function(d) { return d.x; })
+        .attr("cy", function(d) { return d.y; });
+  });
+});

data/lib/aquanaut/templates/index.html.slim

@@ -0,0 +1,29 @@
+doctype html
+html lang="en"
+  head
+    meta charset="UTF -8"
+    title Sitemap #{@domain}
+    script type="text/javascript" src="assets/js/jquery-2.1.0.min.js"
+    script type="text/javascript" src="assets/js/bootstrap.min.js"
+    script type="text/javascript" src="assets/js/d3.v3.min.js"
+    script type="text/javascript" src="assets/js/graph.js"
+
+    script type="text/javascript"
+      | window.graph = #{@graph.to_json}
+
+    link rel="stylesheet" href="assets/css/bootstrap.min.css"
+    link rel="stylesheet" href="assets/css/custom.css"
+  body
+    div class="container"
+      h1 Sitemap for #{@domain}
+      h2 Visualized
+      div class="text-center" id="graph"
+      h2 Overview
+      ul
+        - @graph.select { |node, _| node.instance_of?(Aquanaut::PageNode) }.each do |node, adjacency|
+          li
+            a href="#{node.uri}" #{node.display}
+          ul
+            - adjacency.select { |node| node.instance_of?(Aquanaut::PageNode) }.each do |node|
+              li
+                a href="#{node.uri}" #{node.display}

data/lib/aquanaut/version.rb

@@ -0,0 +1,4 @@
+module Aquanaut
+  # Version of this gem
+  VERSION = "0.1.1"
+end

data/lib/aquanaut/worker.rb

@@ -0,0 +1,111 @@
+require 'mechanize'
+require 'public_suffix'
+
+# The worker contains the actual crawling procedure.
+#
+class Aquanaut::Worker
+
+  def initialize(target)
+    uri = URI.parse(target)
+    @queue = [uri]
+    @domain = PublicSuffix.parse(uri.host)
+
+    @visited = Hash.new(false)
+
+    @agent = Mechanize.new do |agent|
+      agent.open_timeout = 5
+      agent.read_timeout = 5
+    end
+  end
+  
+  # Triggers the crawling process.
+  #
+  def explore
+    while not @queue.empty?
+      uri = @queue.shift  # dequeue
+      next if @visited[uri]
+
+      @visited[uri] = true
+      puts "Visit #{uri}"
+
+      links, assets = links(uri)
+      links.each do |link|
+        @queue.push(link) unless @visited[link]  # enqueue
+      end
+
+      yield uri, links, assets if block_given?
+    end
+  end
+
+  # Retrieves all links to pages and static assets from a given page. The
+  # decision whether a link points to an internal or external domain cannot be
+  # done by just exmaining the link's URL. Due to possible HTTP 3xx responses
+  # the link needs to be resolved. Hence, each link is processed through a HTTP
+  # HEAD request to retrieve the final location.
+  #
+  # @param uri [URI] the URI from which the page is retrieved.
+  #
+  # @return [Array<URI>, Array<Hash>] list of links and static assets found on
+  #                                   the given page.
+  #
+  def links(uri)
+    page = @agent.get(uri)
+    grabbed = Hash.new(false)
+    return [] unless page.is_a?(Mechanize::Page)
+
+    assets = page.images.map do |image|
+      uri = URI.join(page.uri, image.url)
+      { 'uri' => uri, 'type' => 'image' }
+    end
+
+    page.parser.css('link[rel="stylesheet"]').each do |stylesheet|
+      uri = URI.join(page.uri, stylesheet['href'])
+      asset = { 'uri' => uri, 'type' => 'styleshet' }
+      assets << asset
+    end
+
+    links = page.links.map do |link|
+      begin
+        next if link.uri.nil?
+        reference = URI.join(page.uri, link.uri)
+
+        next if grabbed[reference]
+        header = @agent.head(reference)
+
+        location = header.uri
+        next if not internal?(location) or not header.is_a?(Mechanize::Page)
+
+        grabbed[reference] = true
+        grabbed[location] = true
+
+        location
+      rescue Mechanize::Error, URI::InvalidURIError,
+        Net::HTTP::Persistent::Error, Net::OpenTimeout, Net::ReadTimeout
+        next
+      end
+    end.compact
+
+    return links, assets
+  rescue Mechanize::Error, Net::OpenTimeout, Net::ReadTimeout,
+    Net::HTTP::Persistent::Error
+    return [], []  # swallow
+  end
+
+  # Evaluates if a link stays in the initial domain.
+  #
+  # Used to keep the crawler inside the initial domain. In order to determinate
+  # it uses the second-level and top-level domain. If the public suffix cannot
+  # be detected due to possibly invalidity returns true to make sure the link
+  # does not go unchecked.
+  # 
+  # @param link [URI] the link to be checked.
+  #
+  # @return [Boolean] whether the link is internal or not.
+  #
+  def internal?(link)
+    return true unless PublicSuffix.valid?(link.host)
+    link_domain = PublicSuffix.parse(link.host)
+    @domain.sld == link_domain.sld and @domain.tld == link_domain.tld
+  end
+
+end

data/lib/aquanaut.rb

@@ -0,0 +1,41 @@
+require 'aquanaut/asset_node'
+require 'aquanaut/graph'
+require 'aquanaut/page_node'
+require 'aquanaut/sitemap'
+require 'aquanaut/version'
+require 'aquanaut/worker'
+
+# Main module of Aquanaut
+#
+module Aquanaut
+  class << self
+
+    # Processes the given target domain and creates a page and asset graph.
+    #
+    # @param [String] target_address
+    #
+    # @return [Graph] the sitemap graph with pages and static assets
+    #
+    def process_domain(target_address)
+      worker = Worker.new(target_address)
+      graph = Graph.new
+
+      worker.explore do |page_uri, links, static_assets|
+        graph.add_node(PageNode.new(page_uri))
+
+        links.each do |link_uri|
+          graph.add_node(PageNode.new(link_uri))
+          graph.add_edge(page_uri, link_uri)
+        end
+
+        static_assets.each do |asset|
+          graph.add_node(AssetNode.new(asset['uri'], asset['type']))
+          graph.add_edge(page_uri, asset['uri'])
+        end
+      end
+
+      return graph
+    end
+
+  end
+end