aquanaut 0.1.1

data/spec/aquanaut/aquanaut_spec.rb

@@ -0,0 +1,48 @@
+require 'aquanaut'
+require 'spec_helper'
+require 'webmock/rspec'
+
+describe Aquanaut do
+
+  describe ".process_domain" do
+    it "builds a graph with pages as nodes and interlinks as edges" do
+      body = <<-BODY
+      <a href="/home.html">Home</a>
+      <a href="/about.html">About us</a>
+      BODY
+
+      uri = URI.parse('http://www.example.com')
+
+      response = { body: body, headers: { 'Content-Type' => 'text/html'} }
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      stub_request(:head, 'www.example.com/home.html').to_return(response)
+      stub_request(:get, 'www.example.com/home.html').to_return(response)
+
+      stub_request(:head, 'www.example.com/about.html').to_return(response)
+      stub_request(:get, 'www.example.com/about.html').to_return(response)
+
+      graph = Aquanaut.process_domain('http://www.example.com')
+      
+      uris = ['http://www.example.com/home.html',
+              'http://www.example.com/about.html'].map { |u| URI.parse(u) }
+
+      root_node = graph[uri]
+      page_1_node = graph[uris[0]]
+      page_2_node = graph[uris[1]]
+
+      expect(root_node).to be_an_instance_of(Aquanaut::PageNode)
+      expect(page_1_node).to be_an_instance_of(Aquanaut::PageNode)
+      expect(page_2_node).to be_an_instance_of(Aquanaut::PageNode)
+
+      adjacency_list = [page_1_node, page_2_node]
+      expect(root_node.adjacency_list).to eq(adjacency_list)
+
+      expect(page_1_node.adjacency_list).to eq(adjacency_list)
+      expect(page_2_node.adjacency_list).to eq(adjacency_list)
+    end
+  end
+
+end

data/spec/aquanaut/asset_node_spec.rb

@@ -0,0 +1,16 @@
+require 'aquanaut'
+require 'spec_helper'
+
+describe Aquanaut::AssetNode do
+
+  describe "#initialize" do
+    it "stores the URI and type" do
+      uri = URI.parse('http://www.example.com/picture.jpg')
+      node = Aquanaut::AssetNode.new(uri, :image)
+
+      expect(node.uri).to eq(uri)
+      expect(node.type).to eq(:image)
+    end
+  end
+
+end

data/spec/aquanaut/graph_spec.rb

@@ -0,0 +1,89 @@
+require 'aquanaut'
+require 'spec_helper'
+
+describe Aquanaut::Graph do
+
+  describe "#initialize" do
+    it "initializes an empty nodes hash" do
+      graph = Aquanaut::Graph.new
+      expect(graph.instance_variable_get('@nodes')).to be_empty
+    end
+  end
+
+  describe "#add_node" do
+    it "adds the node and hashes the node based on its URI attribute" do
+      uri = URI.parse('http://www.example.com')
+      node = Aquanaut::PageNode.new(uri)
+
+      graph = Aquanaut::Graph.new
+      graph.add_node(node)
+
+      expect(graph[node.uri]).to eq(node)
+    end
+
+    it "does not add a node if it already exists under the given URI" do
+      uri = URI.parse('http://www.example.com')
+      node = Aquanaut::PageNode.new(uri)
+      same_node = Aquanaut::PageNode.new(uri)
+
+      graph = Aquanaut::Graph.new
+      graph.add_node(node)
+
+      expect(graph[uri]).to be(node)
+      expect(graph[uri]).to_not be(same_node)
+    end
+  end
+
+  describe "#add_edge" do
+    it "looks up the predecessor node and delegates to the node's method" do
+      uri = URI.parse('http://www.example.com')
+      node = Aquanaut::PageNode.new(uri)
+
+      adjacent_uri = URI.parse('http://www.example.com/home.html')
+      adjacent_node = Aquanaut::PageNode.new(adjacent_uri)
+
+      graph = Aquanaut::Graph.new
+      graph.add_node(node)
+      graph.add_node(adjacent_node)
+      graph.add_edge(uri, adjacent_uri)
+
+      expect(graph[uri].adjacency_list.first).to eq(adjacent_node)
+    end
+  end
+
+  describe "#[]" do
+    it "looks up a node based on its URI" do
+      uri = URI.parse('http://www.example.com')
+      node = Aquanaut::PageNode.new(uri)
+
+      graph = Aquanaut::Graph.new
+      graph.add_node(node)
+
+      expect(graph[uri]).to be(node)
+    end
+  end
+
+  describe "#each" do
+    it "returns the node object and its adjacency list in an iteration" do
+      uri = URI.parse('http://www.example.com')
+      node = Aquanaut::PageNode.new(uri)
+
+      adjacent_uri = URI.parse('http://www.example.com/home.html')
+      adjacent_node = Aquanaut::PageNode.new(adjacent_uri)
+
+      graph = Aquanaut::Graph.new
+      graph.add_node(node)
+      graph.add_node(adjacent_node)
+      graph.add_edge(uri, adjacent_uri)
+
+      graph.each do |page_node, adjacency_list|
+        expect(page_node).to be_an_instance_of(Aquanaut::PageNode)
+        expect(page_node).to be(node)
+        expect(adjacency_list).to be(node.adjacency_list)
+        break
+      end
+
+    end
+  end
+
+end

data/spec/aquanaut/node_spec.rb

@@ -0,0 +1,26 @@
+require 'aquanaut'
+require 'spec_helper'
+
+describe Aquanaut::Node do
+
+  describe "#initialize" do
+    it "initializes an empty adjacency list" do
+      node = Aquanaut::Node.new
+      expect(node.adjacency_list).to be_empty
+    end
+  end
+
+  describe "#add_edge" do
+    it "adds a successor to the adjacency list" do
+      node = Aquanaut::Node.new
+      adjacent_node = Aquanaut::Node.new
+
+      expect do
+        node.add_edge(adjacent_node)
+      end.to change { node.adjacency_list.count }.by(1)
+
+      expect(node.adjacency_list.first).to eq(adjacent_node)
+    end
+  end
+
+end

data/spec/aquanaut/page_node_spec.rb

@@ -0,0 +1,14 @@
+require 'aquanaut'
+require 'spec_helper'
+
+describe Aquanaut::PageNode do
+
+  describe "#initialize" do
+    it "stores the URI" do
+      uri = URI.parse('http://www.example.com')
+      node = Aquanaut::PageNode.new(uri)
+      expect(node.uri).to eq(uri)
+    end
+  end
+
+end

data/spec/aquanaut/sitemap_spec.rb

@@ -0,0 +1,60 @@
+require 'aquanaut'
+require 'spec_helper'
+
+describe Aquanaut::Sitemap do
+  
+  describe "#initialize" do
+    it "stores the given graph and domain" do
+      graph = Aquanaut::Graph.new
+      domain = 'http://www.example.com'
+      sitemap = Aquanaut::Sitemap.new(graph, domain)
+
+      expect(sitemap.instance_variable_get('@graph')).to be(graph)
+      expect(sitemap.instance_variable_get('@domain')).to be(domain)
+    end
+
+    it "expands the path of the given target directory" do
+      graph = Aquanaut::Graph.new
+      domain = 'http://www.example.com'
+      target_dir = 'spec/sitemap'
+
+      sitemap = Aquanaut::Sitemap.new(graph, domain, target_dir)
+      expanded_dir = sitemap.instance_variable_get('@target_dir')
+
+      expect(Pathname.new(expanded_dir).absolute?).to be_true
+      expect(expanded_dir.end_with?("/aquanaut/#{target_dir}")).to be_true
+    end
+  end
+
+  describe "#initialize_target_directory" do
+    it "creates the directory and copies assets file if neccessary" do
+      graph = Aquanaut::Graph.new
+      domain = 'http://www.example.com'
+      target_dir = 'spec/sitemap'
+
+      sitemap = Aquanaut::Sitemap.new(graph, domain, target_dir)
+      sitemap.send(:initialize_target_directory)
+
+      expect(Dir.exist?(target_dir)).to be_true
+      expect(Dir.exist?("#{target_dir}/assets")).to be_true
+
+      FileUtils.rm_r(target_dir)
+    end
+  end
+
+  describe "#render_results" do
+    it "the result rendering works for an empty graph" do
+      graph = Aquanaut::Graph.new
+      domain = 'http://www.example.com'
+      target_dir = 'spec/sitemap'
+
+      sitemap = Aquanaut::Sitemap.new(graph, domain, target_dir)
+      sitemap.render_results
+
+      expect(File.exist?("#{target_dir}/index.html"))
+
+      FileUtils.rm_r(target_dir)
+    end
+  end
+
+end

data/spec/aquanaut/worker_spec.rb

@@ -0,0 +1,308 @@
+require 'aquanaut'
+require 'spec_helper'
+require 'webmock/rspec'
+
+describe Aquanaut::Worker do
+  describe "#initialize" do
+    it "initializes the queue with the target address" do
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      queue = worker.instance_variable_get('@queue')
+      expected_queue = [URI.parse(target)]
+      expect(queue).to eq(expected_queue)
+    end
+
+    it "stores the target address in its different components" do
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      domain = worker.instance_variable_get('@domain')
+      expect(domain.tld).to eq('com')
+      expect(domain.sld).to eq('example')
+      expect(domain.trd).to eq('www')
+    end    
+  end
+
+  describe "#internal?" do
+    it "compares second-level and top-level domain" do
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      uri = URI.parse('http://www.example.com')
+      expect(worker.internal?(uri)).to be_true
+
+      uri = URI.parse('http://blog.example.com')
+      expect(worker.internal?(uri)).to be_true
+
+      uri = URI.parse('http://www.not-example.com')
+      expect(worker.internal?(uri)).to be_false
+    end
+
+    it "guards against invalid domains" do
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      uri = URI.parse('/internal.html')
+      expect(worker.internal?(uri)).to be_true
+    end
+  end
+
+  describe "#links" do
+    it "retrieves no links from a page with no body" do
+      response = { headers: { 'Content-Type' => 'text/html'} }
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      uri = URI.parse('http://www.example.com')
+      expect(worker.links(uri).first).to be_empty
+    end
+
+    it "returns a list of URIs for a page with anchor elements" do
+      body = <<-BODY
+      <a href="/home.html">Home</a>
+      <a href="/about.html">About us</a>
+      <a href="/contact.html">Contact</a>
+      BODY
+
+      response = { body: body, headers: { 'Content-Type' => 'text/html'} }
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      stub_request(:head, 'www.example.com/home.html').to_return(response)
+      stub_request(:get, 'www.example.com/home.html').to_return(response)
+
+      stub_request(:head, 'www.example.com/about.html').to_return(response)
+      stub_request(:get, 'www.example.com/about.html').to_return(response)
+
+      stub_request(:head, 'www.example.com/contact.html').to_return(response)
+      stub_request(:get, 'www.example.com/contact.html').to_return(response)
+
+      stub_request(:head, 'www.not-example.com').to_return(response)
+
+      uris = ['http://www.example.com/home.html',
+              'http://www.example.com/about.html',
+              'http://www.example.com/contact.html']
+
+      uris.map! { |uri| URI.parse(uri) }
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      uri = URI.parse('http://www.example.com')
+      expect(worker.links(uri).first).to eq(uris)
+    end
+
+    it "returns the final location when encountering HTTP 3xx" do
+      body = '<a href="http://follow-me.com">Follow me</a>'
+      response = { body: body, headers: { 'Content-Type' => 'text/html'} }
+      stub_request(:get, 'www.example.com').to_return(response)
+    end
+
+    it "filters links that reference an external domain directly" do
+      body = <<-BODY
+      <a href="/home.html">Home</a>
+      <a href="/about.html">About us</a>
+      <a href="/contact.html">Contact</a>
+      <a href="http://www.not-example.com">Not Example</a>
+      BODY
+
+      response = { body: body, headers: { 'Content-Type' => 'text/html'} }
+
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      stub_request(:head, 'www.example.com/home.html').to_return(response)
+      stub_request(:get, 'www.example.com/home.html').to_return(response)
+
+      stub_request(:head, 'www.example.com/about.html').to_return(response)
+      stub_request(:get, 'www.example.com/about.html').to_return(response)
+
+      stub_request(:head, 'www.example.com/contact.html').to_return(response)
+      stub_request(:get, 'www.example.com/contact.html').to_return(response)
+
+      stub_request(:head, 'www.not-example.com').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      uris = ['http://www.example.com/home.html',
+              'http://www.example.com/about.html',
+              'http://www.example.com/contact.html']
+
+      uris.map! { |uri| URI.parse(uri) }
+
+      uri = URI.parse('http://www.example.com')
+      expect(worker.links(uri).first).to eq(uris)
+    end
+
+    it "filters links that reference an external domain indirectly" do
+        body = <<-BODY
+        <a href="/home.html">Home</a>
+        <a href="/about.html">About us</a>
+        <a href="/contact.html">Contact</a>
+        <a href="/moved.html">Moved</a>
+        BODY
+
+        other_domain = 'http://www.not-example.com'
+        response = { body: body, headers: { 'Content-Type' => 'text/html'} }
+        forward = { status: 301, headers: { 'Location' => other_domain } }
+
+        stub_request(:get, 'www.example.com').to_return(response)
+
+        stub_request(:head, 'www.example.com/home.html').to_return(response)
+        stub_request(:get, 'www.example.com/home.html').to_return(response)
+
+        stub_request(:head, 'www.example.com/about.html').to_return(response)
+        stub_request(:get, 'www.example.com/about.html').to_return(response)
+
+        stub_request(:head, 'www.example.com/contact.html').to_return(response)
+        stub_request(:get, 'www.example.com/contact.html').to_return(response)
+
+        stub_request(:head, 'www.example.com/moved.html').to_return(forward)
+        stub_request(:head, other_domain).to_return(response)
+
+        target = 'http://www.example.com'
+        worker = Aquanaut::Worker.new(target)
+
+        uris = ['http://www.example.com/home.html',
+                'http://www.example.com/about.html',
+                'http://www.example.com/contact.html']
+
+        uris.map! { |uri| URI.parse(uri) }
+
+        uri = URI.parse('http://www.example.com')
+        expect(worker.links(uri).first).to eq(uris)
+    end
+
+    it "rejects errors raised by Mechanize when retrieving the page" do
+      response = { status: 500 }
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+      uri = URI.parse(target)
+
+      expect(worker.links(uri).first).to be_empty
+    end
+
+    it "rejects errors raised by Mechanize when checking the links" do
+      body = <<-BODY
+      <a href="/home.html">Home</a>
+      <a href="/about.html">About us</a>
+      BODY
+
+      headers = { 'Content-Type' => 'text/html'}
+
+      response = { body: body, headers: headers }
+      response_500 = { status: 500 }
+
+      stub_request(:get, 'www.example.com').to_return(response)
+      stub_request(:head, 'www.example.com/home.html').to_return(response)
+      stub_request(:head, 'www.example.com/about.html').to_return(response_500)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+      uri = URI.parse(target)
+
+      uris = [URI.parse('http://www.example.com/home.html')]
+      expect(worker.links(uri).first).to eq(uris)
+    end
+
+    it "rejects invalid URIs" do
+      body = '<a href="http:invalid.com">Invalid</a>'
+
+      headers = { 'Content-Type' => 'text/html'}
+      response = { body: body, headers: headers }
+
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+      uri = URI.parse(target)
+
+      expect(worker.links(uri).first).to be_empty
+    end
+
+    it "rejects anchors with no href attribute" do
+      body = '<a>Empty</a>'
+
+      headers = { 'Content-Type' => 'text/html'}
+      response = { body: body, headers: headers }
+
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+      uri = URI.parse(target)
+
+      expect(worker.links(uri).first).to be_empty
+    end
+
+    it "rejects links that lead to a timeout" do
+      body = '<a href="/timeout.html">Timeout</a>'
+
+      headers = { 'Content-Type' => 'text/html'}
+      response = { body: body, headers: headers }
+
+      stub_request(:get, 'www.example.com').to_return(response)
+      stub_request(:head, 'www.example.com/timeout.html').to_timeout
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+      uri = URI.parse(target)
+
+      expect(worker.links(uri).first).to be_empty
+    end
+
+    it "rejects links that have already been grabbed" do
+      body = <<-BODY
+      <a href="/home.html">Home</a>
+      <a href="/home.html">Home</a>
+      BODY
+
+      response = { body: body, headers: { 'Content-Type' => 'text/html'} }
+
+      stub_request(:get, 'www.example.com').to_return(response)
+      stub_request(:get, 'www.example.com/home.html').to_return(response)
+      stub_request(:head, 'www.example.com/home.html').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+      uri = URI.parse(target)
+
+      result = [URI.parse('http://www.example.com/home.html')]
+      expect(worker.links(uri).first).to eq(result)
+    end
+  end
+
+  describe "#explore" do
+    it "starts the crawling by processing the first queue element" do
+      response = { headers: { 'Content-Type' => 'text/html'} }
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+      worker.explore
+
+      queue = worker.instance_variable_get('@queue')
+      expect(queue).to be_empty
+    end
+
+    it "marks visited sites" do
+      response = { headers: { 'Content-Type' => 'text/html'} }
+      stub_request(:get, 'www.example.com').to_return(response)
+
+      target = 'http://www.example.com'
+      worker = Aquanaut::Worker.new(target)
+
+      visited = worker.instance_variable_get('@visited')
+      expect { worker.explore }.to change { visited.size }.by(1)
+    end
+
+    it "skips already visited sites" do
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end