ruby-crawler 0.0.1

data/lib/crawler/validations.rb

@@ -0,0 +1,12 @@
+module Crawler
+  module Validations
+
+    VALID_PROTOCOLS = ['http', 'https'].freeze
+
+    # Validates protocol is http or https
+    #
+    def validate_protocol
+      raise InvalidProtocolError.new('Please specify either http or https') unless VALID_PROTOCOLS.include? base_uri.scheme
+    end
+  end
+end

data/lib/crawler/version.rb

@@ -0,0 +1,3 @@
+module Crawler
+  VERSION = "0.0.1"
+end

data/test/crawler/crawler_test.rb

@@ -0,0 +1,46 @@
+require 'test_helper'
+
+class CrawlerTest < MiniTest::Test
+
+  def test_initialization
+    crawler = Crawler.new('https://www.digitalocean.com')
+    assert_equal 'www.digitalocean.com', crawler.base_uri.hostname
+    assert_equal 'https://www.digitalocean.com', crawler.base_url
+    assert_equal 'https', crawler.base_uri.scheme
+  end
+
+  def test_crawl
+    crawler = Crawler.new('https://www.digitalocean.com')
+    crawler.crawl
+    results = crawler.results
+    assert_equal expected_crawl_results, results
+  end
+
+  def expected_crawl_results
+    {
+      "domain" => "www.digitalocean.com",
+      "paths" => {
+        "/company/careers/" => {
+          "asset_dependencies" => [],
+          "links_to" => [],
+          "linked_to_from" => ["/features/", "/pricing/", "/"]
+        },
+        "/" => {
+          "asset_dependencies" => ["//use.typekit.net/wix0mlm.js", "/assets/css/style.css", "/assets/images/cover_create.jpg", "/assets/images/techcrunch.png"],
+          "links_to" => ["/company/careers/",  "/pricing/", "/"],
+          "linked_to_from" => ["/features/", "/pricing/"]
+        },
+        "/pricing/" => {
+            "asset_dependencies" => ["/assets/css/style.css", "/assets/images/mashable.png"],
+            "links_to" => ["/company/careers/", "/features/", "/"],
+            "linked_to_from" => ["/features/", "/"]
+        },
+        "/features/" => {
+          "asset_dependencies" => ["/assets/images/venturebeat.png"],
+          "links_to" => ["/company/careers/", "/pricing/", "/"],
+          "linked_to_from" => ["/pricing/"]
+        }
+      }
+    }
+  end
+end

data/test/crawler/document_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+
+class DocumentTest < MiniTest::Test
+
+  def test_document_parsing
+    document = Crawler::Document.new('https://www.digitalocean.com/')
+
+    # Test link discovery
+    ['http://techcrunch.com/article/', 'https://cloud.digitalocean.com/login', '/company/careers/'].each do |link|
+      assert_includes document.links, link
+    end
+
+    # Test domain specific link discovery
+    assert_includes document.domain_specific_paths, '/company/careers/'
+
+    ['https://twitter.com/digitalocean', 'https://status.digitalocean.com/', '#'].each do |link|
+      refute_includes document.domain_specific_paths, link
+    end
+
+    # Test assest discovery
+    assert_includes document.static_assets, '/assets/images/techcrunch.png'
+  end
+end

data/test/crawler/formatting_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+require 'crawler/formatting'
+
+class FormattingTest < MiniTest::Test
+  include Crawler::Formatting
+
+  def test_normalize_path
+    path = '/'
+    assert_equal '/', normalize_path(path)
+
+    path = '/pricing'
+    assert_equal '/pricing/', normalize_path(path)
+
+    path = '/pricing/'
+    assert_equal '/pricing/', normalize_path(path)
+  end
+
+  def test_construct_url
+    url = 'https://www.digitalocean.com'
+    uri = URI.parse(url)
+    assert_equal url, construct_url(uri)
+  end
+end

data/test/crawler/http_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+
+class HttpTest < MiniTest::Test
+  include Crawler::Http
+
+  def test_request
+    domain = 'https://www.digitalocean.com'
+    response = request domain
+    assert_equal 'SSD Cloud Server, VPS Server, Simple Cloud Hosting | DigitalOcean', Nokogiri::HTML(response).css('title').text
+  end
+
+  def test_request_follows_redirect
+    domain = 'https://www.digitalocean.com/redirect'
+    response = request domain
+    assert_equal 'SSD Cloud Server, VPS Server, Simple Cloud Hosting | DigitalOcean', Nokogiri::HTML(response).css('title').text
+  end
+
+  def test_bad_request_is_rescued
+    domain = 'https://www.digitalocean.com/bad_request'
+    response = request domain
+    assert_equal '', response
+  end
+end

data/test/crawler/index_test.rb

@@ -0,0 +1,78 @@
+require 'test_helper'
+
+class IndexTest < MiniTest::Test
+  include Crawler::Formatting
+
+  def test_initialization
+    domain = 'https://www.digitalocean.com'
+    index = Crawler::Index.new(Addressable::URI.parse(domain))
+
+    expected_results = {
+      'domain' => 'www.digitalocean.com',
+      'paths' => {}
+    }
+
+    assert_equal expected_results, index.results
+  end
+
+  def test_document_consumption
+    domain = "https://www.digitalocean.com"
+    document = Crawler::Document.new(domain)
+    uri = Addressable::URI.parse(domain)
+    index = Crawler::Index.new(uri)
+    index.consume_document uri.path, document
+
+    # Test domain is correct
+    assert_equal 'www.digitalocean.com', index.base_uri.hostname
+
+    # Test that urls / links are properly recorded
+    results = index.results
+
+    ["/company/careers/", "/", "/pricing/"].each do |key|
+      assert_includes results['paths'].keys, key
+    end
+
+    paths_to_visit = index.get_paths_to_visit
+    ["/company/careers/", "/pricing/"].each do |path|
+      assert_includes paths_to_visit, path
+    end
+
+    paths = index.get_stored_paths
+    ["/"].each do |path|
+      assert_includes paths, path
+    end
+
+    # Test that the assets are recorded
+    assets = index.get_path_assets('/')
+    ["/assets/images/techcrunch.png", "/assets/images/cover_create.jpg", "/assets/css/style.css"].each do |asset|
+      assert_includes assets, asset
+    end
+
+    # Test the indexed results
+    assert_equal expected_index_results, index.results
+  end
+
+  def expected_index_results
+    {
+      "domain" => "www.digitalocean.com",
+      "paths" => {
+        "/company/careers/" => {
+          "asset_dependencies" => [],
+          "links_to" => [],
+          "linked_to_from" => ["/"]
+        },
+        "/pricing/" => {
+          "asset_dependencies" => [],
+          "links_to" => [],
+          "linked_to_from"=>["/"]
+        },
+        "/" => {
+          "asset_dependencies" => ["//use.typekit.net/wix0mlm.js", "/assets/css/style.css", "/assets/images/cover_create.jpg", "/assets/images/techcrunch.png"],
+          "links_to" => ["/company/careers/", "/pricing/", "/"],
+          "linked_to_from" => []
+        }
+      }
+    }
+  end
+
+end

data/test/crawler/storage_test.rb

@@ -0,0 +1,147 @@
+require 'test_helper'
+
+class StorageTest < MiniTest::Test
+
+  def test_store_path_and_stored_paths
+    path = '/pricing/'
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path path
+    assert_equal [path], index.get_stored_paths
+  end
+
+  def test_storing_path_assets
+    path = '/pricing/'
+    assets = ['asset1', 'asset2']
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path_assets(path, assets)
+
+    stored_assets = index.get_path_assets(path)
+    assets.each do |asset|
+      assert_includes stored_assets, asset
+    end
+  end
+
+  def test_storing_path_links_to
+    path = '/pricing/'
+    links = ['link1', 'link2']
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path_links_to(path, links)
+
+    stored_links = index.get_path_links_to(path)
+
+    links.each do |link|
+      assert_includes stored_links, link
+    end
+  end
+
+  def test_storing_path_linked_to_from
+    path = '/pricing/'
+    links = ['link1', 'link2']
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path_linked_to_from(path, links)
+
+    stored_links = index.get_path_linked_to_from(path)
+    links.each do |link|
+      assert_includes stored_links, link
+    end
+  end
+
+  def test_storing_paths_visited
+    path = '/pricing/'
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path_visited(path)
+    assert_equal [path], index.get_paths_visited
+  end
+
+  def test_queuing_paths
+    paths = ['/pricing/', '/features/']
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_paths_to_visit(paths)
+    assert_equal paths.reverse, index.get_paths_to_visit
+  end
+
+  def test_get_domain_data
+    path = '/pricing/'
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path path
+
+    assets = ['asset1', 'asset2']
+    index.store_path_assets(path, assets)
+
+    links_to = ['link1', 'link2']
+    index.store_path_links_to(path, links_to)
+
+    linked_to_from = ['link3', 'link4']
+    index.store_path_linked_to_from(path, linked_to_from)
+
+    data = index.get_domain_data
+    assert_equal 'www.digitalocean.com', data['domain']
+
+    assets.each do |value|
+      assert_includes data['paths']['/pricing/']['asset_dependencies'], value
+    end
+
+    links_to.each do |value|
+      assert_includes data['paths']['/pricing/']['links_to'], value
+    end
+
+    linked_to_from.each do |value|
+      assert_includes data['paths']['/pricing/']['linked_to_from'], value
+    end
+  end
+
+  def test_get_path_data
+    path = '/pricing/'
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path path
+
+    assets = ['asset1', 'asset2']
+    index.store_path_assets(path, assets)
+
+    links_to = ['link1', 'link2']
+    index.store_path_links_to(path, links_to)
+
+    linked_to_from = ['link3', 'link4']
+    index.store_path_linked_to_from(path, linked_to_from)
+
+    data = index.get_path_data(path)
+
+    assets.each do |value|
+      assert_includes data['asset_dependencies'], value
+    end
+
+    links_to.each do |value|
+      assert_includes data['links_to'], value
+    end
+
+    linked_to_from.each do |value|
+      assert_includes data['linked_to_from'], value
+    end
+  end
+
+  def test_clear_stored_data
+    path = '/pricing/'
+    index = Crawler::Index.new(Addressable::URI.parse('https://www.digitalocean.com'))
+    index.store_path path
+
+    assets = ['asset1', 'asset2']
+    index.store_path_assets(path, assets)
+
+    links_to = ['link1', 'link2']
+    index.store_path_links_to(path, links_to)
+
+    linked_to_from = ['link3', 'link4']
+    index.store_path_linked_to_from(path, linked_to_from)
+
+    index.clear_stored_results
+
+    empty_results = {
+      'asset_dependencies' => [],
+      'links_to' => [],
+      'linked_to_from' => []
+    }
+
+    assert_equal empty_results, index.get_path_data(path)
+  end
+
+end

data/test/crawler/validations_test.rb

@@ -0,0 +1,10 @@
+require 'test_helper'
+
+class ValidationsTest < MiniTest::Test
+
+  def test_protocol_validation
+    error = assert_raises(Crawler::InvalidProtocolError) { Crawler.new('digitalocean.com') }
+    assert_equal 'Please specify either http or https', error.message
+  end
+
+end

data/test/support/domain_html.rb

@@ -0,0 +1,256 @@
+module DomainHtml
+
+  # Domain links: '/', 'company/careers', '/pricing'
+  # Subdomains: 'cloud.digitalocean.com/...'
+  # External links: 'techcrunch/...'
+  # Assets: /assets/images/techcrunch.png, /assets/images/cover_create.jpg, //use.typekit.net/wix0mlm.js, /assets/css/style.css
+  def root_html
+    '<!doctype html>
+    <html data-placeholder-focus="false">
+      <head>
+        <meta charset="utf-8">
+        <meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible">
+        <title>SSD Cloud Server, VPS Server, Simple Cloud Hosting | DigitalOcean</title>
+        <meta name="description" content="Deploy an 512MB RAM and 20GB SSD cloud server in 55 seconds for $5/month. Simple, fast, scalable SSD cloud virtual servers.">
+        <meta name="keywords" content="cloud server, cloud hosting, vps, vps server, vps server hosting, vps hosting, virtual server, virtual private server, ubuntu server, centos server, debian server, linux server, fedora server, linux mint server">
+        <meta name="viewport" content="width=1200, maximum-scale=1" />
+        <link href="/assets/css/style.css" media="screen" rel="stylesheet" type="text/css" />
+        <script type="text/javascript" src="//use.typekit.net/wix0mlm.js"></script>
+        <script type="text/javascript">try{Typekit.load();}catch(e){}</script>
+      </head>
+      <body class="index">
+        <header>
+          <div class="wrapper-full">
+            <a class="logo" href="/">DigitalOcean</a>
+            <a id="hiring" href="/company/careers/">We\'re Hiring!</a>
+            <nav id="button-nav">
+              <ul>
+                <li><a href="https://cloud.digitalocean.com/login">Log In</a></li>
+                <li class="button-outline"><a href="https://cloud.digitalocean.com/registrations/new">Sign Up</a></li>
+              </ul>
+            </nav>
+            <nav>
+              <ul>
+                <li><a class="" href="/pricing/">Pricing</a></li>
+              </ul>
+            </nav>
+          </div>
+        </header>
+        <section class="main">
+          <section id="homepage-features" class="homepage-section">
+            <div class="wrapper-full">
+              <h2>A control panel experience made simple.</h2>
+              <div id="browser">
+                <div id="last-vid">
+                  <a href="#" class="restart standard-button">Replay</a>
+                </div>
+                <p class="url">https://cloud.digitalocean.com/droplets</p>
+                <video height="100%" class="feature-video" preload="none" poster="/assets/images/cover_create.jpg">
+                  <source src="/assets/video/create.mp4" type="video/mp4">
+                  <source src="/assets/video/create.webm" type="video/webm">
+                  Your browser does not support the video tag.
+                </video>
+              </div>
+              <ul id="feature-controls">
+                <li class="create active" data-video-url="/assets/video/create.webm" data-url-bar="https://cloud.digitalocean.com/droplets">
+                  <a href="#">
+                    <div class="icon">
+                      <span></span>
+                    </div>
+                    <h3>Create Droplets</h3>
+                    <p>Setup and deploy your Droplet\'s configuration in a matter of seconds. </p>
+                  </a>
+                </li>
+              </ul>
+            </div>
+          </section>
+          <section id="companies-using">
+            <div class="wrapper-full">
+              <h3>As Seen On</h3>
+              <ul>
+                <li><a href="http://techcrunch.com/article/"><img src="/assets/images/techcrunch.png"></a></li>
+              </ul>
+            </div>
+          </section>
+          <section class="call-to-action">
+            <div class="wrapper-full">
+               <h2>Deploy an SSD cloud server in 55 seconds.</h2>
+               <p>Sign up for DigitalOcean today.</p>
+               <a class="standard-button" href="https://cloud.digitalocean.com/registrations/new">
+                 GET STARTED NOW <span></span>
+               </a>
+            </div>
+          </section>
+        </section>
+      </body>
+    </html>'
+  end
+
+  # Domain links: '/', 'company/careers', '/features'
+  # Subdomains: 'cloud.digitalocean.com/...'
+  # External links: 'mashable/...'
+  # Assets: /assets/css/style.css, /assets/images/mashable.png
+  def pricing_html
+    '<!doctype html>
+    <html data-placeholder-focus="false">
+      <head>
+        <meta charset="utf-8">
+        <meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible">
+        <title>Pricing</title>
+        <meta name="description" content="Deploy an 512MB RAM and 20GB SSD cloud server in 55 seconds for $5/month. Simple, fast, scalable SSD cloud virtual servers.">
+        <meta name="keywords" content="cloud server, cloud hosting, vps, vps server, vps server hosting, vps hosting, virtual server, virtual private server, ubuntu server, centos server, debian server, linux server, fedora server, linux mint server">
+        <meta name="viewport" content="width=1200, maximum-scale=1" />
+        <link href="/assets/css/style.css" media="screen" rel="stylesheet" type="text/css" />
+        <script type="text/javascript">try{Typekit.load();}catch(e){}</script>
+      </head>
+      <body class="index">
+        <header>
+          <div class="wrapper-full">
+            <a class="logo" href="/">DigitalOcean</a>
+            <a id="hiring" href="/company/careers/">We\'re Hiring!</a>
+            <nav id="button-nav">
+              <ul>
+                <li><a href="https://cloud.digitalocean.com/login">Log In</a></li>
+                <li class="button-outline"><a href="https://cloud.digitalocean.com/registrations/new">Sign Up</a></li>
+              </ul>
+            </nav>
+            <nav>
+              <ul>
+                <li><a class="" href="/features/">Features</a></li>
+                <li><a id="hiring" href="/company/careers/">We\'re Hiring!</a></li>
+              </ul>
+            </nav>
+          </div>
+        </header>
+        <section class="main">
+          <section id="homepage-features" class="homepage-section">
+            <div class="wrapper-full">
+              <h2>A control panel experience made simple.</h2>
+              <div id="browser">
+                <div id="last-vid">
+                  <a href="#" class="restart standard-button">Replay</a>
+                </div>
+                <p class="url">https://cloud.digitalocean.com/droplets</p>
+              </div>
+              <ul id="feature-controls">
+                <li class="dns" data-video-url="/assets/video/dns.webm" data-url-bar="https://cloud.digitalocean.com/domains">
+                  <a href="#">
+                    <div class="icon">
+                      <span></span>
+                    </div>
+                    <h3>DNS Management</h3>
+                    <p>Full feature DNS management allows<br>you to easily manage your domains.</p>
+                  </a>
+                </li>
+              </ul>
+            </div>
+          </section>
+          <section id="companies-using">
+            <div class="wrapper-full">
+              <h3>As Seen On</h3>
+              <ul>
+                <li><a href="http://mashable.com/article/"><img src="/assets/images/mashable.png"></a></li>
+              </ul>
+            </div>
+          </section>
+          <section class="call-to-action">
+            <div class="wrapper-full">
+               <h2>Deploy an SSD cloud server in 55 seconds.</h2>
+               <p>Sign up for DigitalOcean today.</p>
+               <a class="standard-button" href="https://cloud.digitalocean.com/registrations/new">
+                 GET STARTED NOW <span></span>
+               </a>
+            </div>
+          </section>
+        </section>
+      </body>
+    </html>'
+  end
+
+  # Domain links: '/', 'company/careers', '/pricing'
+  # Subdomains: 'cloud.digitalocean.com/...'
+  # External links: 'venturebeat/...'
+  # Assets: /assets/images/venturebeat.png
+  def features_html
+    '<!doctype html>
+    <html data-placeholder-focus="false">
+      <head>
+        <meta charset="utf-8">
+        <meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible">
+        <title>Features</title>
+        <meta name="description" content="Deploy an 512MB RAM and 20GB SSD cloud server in 55 seconds for $5/month. Simple, fast, scalable SSD cloud virtual servers.">
+        <meta name="keywords" content="cloud server, cloud hosting, vps, vps server, vps server hosting, vps hosting, virtual server, virtual private server, ubuntu server, centos server, debian server, linux server, fedora server, linux mint server">
+        <meta name="viewport" content="width=1200, maximum-scale=1" />
+        <script type="text/javascript">try{Typekit.load();}catch(e){}</script>
+      </head>
+      <body class="index">
+        <header>
+          <div class="wrapper-full">
+            <a class="logo" href="/">DigitalOcean</a>
+            <a id="hiring" href="/company/careers/">We\'re Hiring!</a>
+            <nav id="button-nav">
+              <ul>
+                <li><a href="https://cloud.digitalocean.com/login">Log In</a></li>
+                <li class="button-outline"><a href="https://cloud.digitalocean.com/registrations/new">Sign Up</a></li>
+              </ul>
+            </nav>
+            <nav>
+              <ul>
+                <li><a class="" href="/pricing/">Pricing</a></li>
+              </ul>
+            </nav>
+          </div>
+        </header>
+        <section class="main">
+          <section id="homepage-features" class="homepage-section">
+            <div class="wrapper-full">
+              <h2>A control panel experience made simple.</h2>
+              <div id="browser">
+                <div id="last-vid">
+                  <a href="#" class="restart standard-button">Replay</a>
+                </div>
+              </div>
+              <ul id="feature-controls">
+                <li class="backups" data-video-url="/assets/video/snapshot.webm" data-url-bar="https://cloud.digitalocean.com/images">
+                  <a href="#">
+                    <div class="icon">
+                      <span></span>
+                    </div>
+                    <h3>Snapshots</h3>
+                    <p>Build a new server from a snapshot<br>you took, saving you configuration time.</p>
+                  </a>
+                </li>
+              </ul>
+            </div>
+          </section>
+          <section id="companies-using">
+            <div class="wrapper-full">
+              <h3>As Seen On</h3>
+              <ul>
+                <li><a href="http://venturebeat.com/article/"><img src="/assets/images/venturebeat.png"></a></li>
+              </ul>
+            </div>
+          </section>
+          <section class="call-to-action">
+            <div class="wrapper-full">
+               <h2>Deploy an SSD cloud server in 55 seconds.</h2>
+               <p>Sign up for DigitalOcean today.</p>
+               <a class="standard-button" href="https://cloud.digitalocean.com/registrations/new">
+                 GET STARTED NOW <span></span>
+               </a>
+            </div>
+          </section>
+        </section>
+      </body>
+    </html>'
+  end
+
+  def careers_html
+    '<!doctype html>
+    <html data-placeholder-focus="false">
+      <body>
+        No substantial content
+      </body>
+    </html>'
+  end
+end