anaximander 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 009aaa84665dcd8497a59b9e47acef29008613ea
+  data.tar.gz: 956c8d4a45e84443af161c5b4c78c4c33f8a54fb
+SHA512:
+  metadata.gz: af827976b24b5f734391e3229a2c6fbd2a05ea012b7766517367d547beca19964bacfd59e96c3a857d12a2ae18d071c6c34f70fe2a7e6354d639b6baab0aa865
+  data.tar.gz: 4e49ba0bb2b67931ac4826f3e2e1913c3ae32885af892226c563488ff069e044c3f36de3c799885381a2581f53f96bb81cb421563874134195f6b9ae328313cd

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Matte Noble
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# Anaximander
+
+Anaximander is a small library for crawling a website and rendering the
+resulting site map to the console.
+
+## Installation & Usage
+
+```sh
+gem install anaximander
+```
+
+```sh
+mapgen <url>
+```
+
+## Running Tests
+
+```sh
+bundle install
+bundle exec rspec spec
+```
+
+### End to End Tests
+
+There are two tests marked with the tag `endtoend`. These tests start up
+a Rack app which serves a simple website and run against that server
+like the library would in "production". Think of them as the Acceptance
+tests for a library.
+
+The end to end tests are run by default. To exclude them:
+
+```sh
+bundle exec rspec spec --tag ~endtoend
+```
+
+## What does Anaximander mean?
+
+Anaximander was a Greek cartographer who was the first person to try to
+map the entire world.
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/anaximander/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/anaximander.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'anaximander/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "anaximander"
+  spec.version       = Anaximander::VERSION
+  spec.authors       = ["Matte Noble"]
+  spec.email         = ["me@mattenoble.com"]
+  spec.summary       = %q{Web scraper that collects assets and links.}
+  spec.description   = %q{Web scraper that collects assets and links.}
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "colorize"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rack"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-autotest"
+  spec.add_development_dependency "fakeout"
+end

data/bin/mapgen

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require "anaximander"
+require "anaximander/cli"
+Anaximander.logger = false
+Anaximander::CLI.start

data/lib/anaximander/cli.rb

@@ -0,0 +1,17 @@
+module Anaximander
+  class CLI
+    def self.start
+      new(ARGV[0]).start
+    end
+
+    def initialize(url)
+      @crawler  = Crawler.new(url)
+      @renderer = Renderer.new
+    end
+
+    def start
+      @crawler.crawl
+      @renderer.draw(@crawler.root)
+    end
+  end
+end

data/lib/anaximander/crawler.rb

@@ -0,0 +1,31 @@
+module Anaximander
+  class Crawler
+    attr_reader :url, :root
+
+    def initialize(url)
+      @url = url.chomp("/")
+      @root = Page.new(url)
+      @visited = [url]
+    end
+
+    def crawl(page=self.root)
+      page.children = page.links.map { |link| visit(link.chomp("/")) }.compact
+      page.children.each { |child| crawl(child) }
+    end
+
+    def visit(link)
+      return if @visited.include?(link)
+
+      logger.debug(link)
+      @visited << link
+
+      Page.new(link)
+    rescue Anaximander::PageNotAccessibleError
+      nil
+    end
+
+    def logger
+      Anaximander.logger
+    end
+  end
+end

data/lib/anaximander/discovery/assets.rb

@@ -0,0 +1,39 @@
+module Anaximander
+  module Discovery
+    class Assets
+      include Enumerable
+      include Comparable
+      extend  Forwardable
+      def_delegators :assets, :size, :inspect, :to_a
+
+      def initialize(page)
+        @page = page
+      end
+
+      def each(&block)
+        assets.each(&block)
+      end
+
+      def <=>(other)
+        to_a <=> other.to_a
+      end
+
+      private
+
+      attr_reader :page
+
+      def assets
+        css + javascript
+      end
+
+      def css
+        page.css("link").map { |link| link[:href] }.compact
+      end
+
+      def javascript
+        page.css("script").map { |script| script[:src] }.compact
+      end
+    end
+  end
+end
+

data/lib/anaximander/discovery/links.rb

@@ -0,0 +1,76 @@
+module Anaximander
+  module Discovery
+
+    # Collection of internal links on the given page. 
+    #
+    # == Relative Paths
+    #
+    # `Anaximander::Discovery::Links` converts all relative paths into absolute
+    # paths using the base URL of the page being crawled.
+    #
+    #   # http://example.com
+    #   <a href="/contact">Contact</a>
+    #
+    #   Anaximander::Discovery::Links.new(Nokogiri::HTML(open("http://example.com")))
+    #   # => ["http://example.com/contact"]
+    #
+    # == Exclusions
+    #
+    #   - External links (ones outside the domain of the page
+    #   - Hash links (Javascript style links with href of "#")
+    #
+    # == Example
+    #
+    #   page = Nokogiri::HTML(open("http://example.com"))
+    #
+    #   Anaximander::Discovery::Links.new(page)
+    #   # => ["http://www.iana.org/domains/example"]
+    #
+    class Links
+      include Enumerable
+      include Comparable
+      extend  Forwardable
+      def_delegators :links, :size, :inspect, :to_a
+
+      # Parameters
+      #
+      #   page [Nokogiri::HTML] Parsed html of the page.
+      #   url  [String|URI] URL of the page to discover.
+      #
+      def initialize(page, url)
+        @page = page
+        @url  = Url.new(url)
+      end
+
+      def each(&block)
+        links.each(&block)
+      end
+
+      def <=>(other)
+        to_a <=> other.to_a
+      end
+
+      private
+
+      attr_reader :page
+
+      def links
+        internal_links.map(&:to_s)
+      end
+
+      def internal_links
+        all_links.select { |link| @url.base == link.base }
+      end
+
+      def all_links
+        page.css("a").map { |a| absolute(a[:href]) }.compact.uniq
+      end
+
+      def absolute(link)
+        Url.new(link).absolute(@url.base).without_fragment
+      rescue URI::InvalidURIError
+        nil
+      end
+    end
+  end
+end

data/lib/anaximander/page.rb

@@ -0,0 +1,73 @@
+module Anaximander
+  class Error < StandardError; end
+
+  # Raised when a page cannot be fetched.
+  #
+  class PageNotAccessibleError < Error; end
+
+  # Represents a single page of a website being crawled. Exposes the assets and 
+  # links on the page.
+  #
+  # == Errors
+  #
+  # `Anaximander::Page` will raise a `PageNotAccessibleError` when the page cannot
+  # be fetched for some reason. This is often due to it not existing (404), SSL
+  # errors or infinite redirect loops.
+  #
+  # == Example
+  #
+  #   page = Page.new("http://example.com")
+  #   page.links  # => ["http://www.iana.org/domains/example"]
+  #   page.assets # => ["/main.css", "/default.js"]
+  #
+  class Page
+    include Comparable
+
+    # Absolute url of the page.
+    #
+    attr_reader :url
+
+    # Parsed Nokogiri HTML document.
+    #
+    attr_reader :html
+
+    # Collection of `Page` objects that are linked 
+    # to from the current page.
+    #
+    attr_accessor :children
+
+    # Parameters
+    #
+    #   [String] url URL to discover.
+    #
+    # OpenURI raises a generic RuntimeError when it cannot fetch a
+    # page, for a variety of reasons. Some of which are 404s, SSL
+    # errors, or redirect loops.
+    #
+    # raises `PageNotAccessibleError` when OpenURI fails to fetch the
+    # page, for any reason.
+    #
+    def initialize(url)
+      @url  = url
+      @html = Nokogiri::HTML(open(url))
+    rescue RuntimeError, OpenURI::HTTPError
+      raise PageNotAccessibleError
+    end
+
+    def links
+      Discovery::Links.new(html, url)
+    end
+
+    def assets
+      Discovery::Assets.new(html)
+    end
+
+    def <=>(other)
+      self.url <=> other.url
+    end
+
+    def inspect
+      %(#<Anaximander::Page:#{object_id} url="#{url}">)
+    end
+  end
+end

data/lib/anaximander/renderer.rb

@@ -0,0 +1,83 @@
+require "colorize"
+
+module Anaximander
+  # Draws the crawled tree of URLs and assets, generated by `Anaximander::Crawler`.
+  #
+  # == Output
+  #
+  #   └── <url> [assets]
+  #       ├── <url> [assets]
+  #       │   └── <url> [assets]
+  #       └── <url> [assets]
+  #
+  # == Example
+  #
+  #   root
+  #   # => #<Anaximander::Page url="http://example.com"/>
+  #
+  #   root.children
+  #   # => [#<Anaximander::Page url="http://example.com/foo"/>, #<Anaximander::Page url="http://example.com"/bar>]
+  #
+  #   renderer = Anaximander::Renderer.new(root)
+  #   renderer.draw
+  #
+  #   # => └── http://example.com [main.css]
+  #   # =>     ├── http://example.com/foo [main.css, foo.js]
+  #   # =>     └── http://example.com/bar [main.css, bar.js]
+  #
+  class Renderer
+    VERTICAL_PIPE = "│   "
+    MEMBER_PIPE   = "├── "
+    TAIL_PIPE     = "└── "
+    SPACE_PIPE    = "    "
+
+    attr_reader :root
+
+    def initialize(options={})
+      @color = options.fetch(:color, true)
+    end
+
+    # Draws a page URL, its assets and recursively does the same for
+    # all of its children.
+    #
+    # == Parameters
+    #
+    #   [Anaximander::Page] page The page to render
+    #   [String] prefix A string that should preceed the actual URLa
+    #                   and asset information.
+    #   [Boolean] tail Is this node the last one in the collection.
+    #
+    def draw(page=self.root, prefix="", tail=true)
+      pipe = tail ? TAIL_PIPE : MEMBER_PIPE
+
+      url    = "#{prefix}#{pipe}#{page.url} "
+      assets = "#{page.assets.to_a}"
+      assets = assets.colorize(:light_black) if @color
+
+      print url
+      puts assets
+
+      page.children[0..-2].each { |child| draw_child(child, prefix, tail) }
+      draw_tail(page.children.last, prefix, tail) if page.children.size >= 1
+    end
+
+    # Draws a child node, with the appropriate "connecting pipe".
+    #
+    # The "connecting pipe" is the character at the beginning of this line,
+    # which connects this to the previous tier of the tree.
+    #
+    def draw_child(page, prefix, parent_is_tail)
+      connecting_pipe = parent_is_tail ? SPACE_PIPE : VERTICAL_PIPE
+      draw(page, "#{prefix}#{connecting_pipe}", false)
+    end
+
+    # Draws a leaf node, with the appropriate "connecting pipe".
+    #
+    # See `draw_child` for "connecting pipe" definition.
+    #
+    def draw_tail(page, prefix, parent_is_tail)
+      connecting_pipe = parent_is_tail ? SPACE_PIPE : VERTICAL_PIPE
+      draw(page, "#{prefix}#{connecting_pipe}", true)
+    end
+  end
+end

data/lib/anaximander/url.rb

@@ -0,0 +1,38 @@
+module Anaximander
+  class Url < SimpleDelegator
+    include Comparable
+
+    attr_reader :uri
+
+    def initialize(uri)
+      @uri = URI(uri.to_s)
+      super(@uri)
+    end
+
+    def base
+      domain  = "#{scheme}://#{host}"
+      domain += ":#{port}" unless port == 80
+      domain
+    end
+
+    def join(url)
+      self.class.new(URI.join(self.uri, url.to_s))
+    end
+
+    def absolute(base)
+      absolute? ? self : Url.new(base).join(self)
+    end
+
+    def without_fragment
+      self.class.new(self).tap { |url| url.fragment = nil }
+    end
+
+    def <=>(other)
+      other.respond_to?(:uri) ? self.uri <=> other.uri : self.uri.to_s <=> other
+    end
+
+    def eql?(other)
+      self.uri.eql?(other.uri)
+    end
+  end
+end

data/lib/anaximander/version.rb

@@ -0,0 +1,3 @@
+module Anaximander
+  VERSION = "0.0.1"
+end

data/lib/anaximander.rb

@@ -0,0 +1,24 @@
+require "open-uri"
+require "nokogiri"
+require "forwardable"
+require "delegate"
+require "uri"
+require "logger"
+
+require "anaximander/version"
+require "anaximander/url"
+require "anaximander/page"
+require "anaximander/crawler"
+require "anaximander/renderer"
+require "anaximander/discovery/links"
+require "anaximander/discovery/assets"
+
+module Anaximander
+  def self.logger=(out)
+    @logger = Logger.new(out)
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+end

data/spec/anaximander/crawler_spec.rb

@@ -0,0 +1,64 @@
+require "spec_helper"
+
+describe Anaximander::Crawler do
+  let(:link)        { "http://example.com" }
+  subject!(:crawler) { described_class.new(link) }
+
+  it "does not visit pages that have already been visited" do
+    crawler.visit(link)
+    expect(crawler.visit(link)).to be_nil
+  end
+
+  it "does not visit pages that do not exist" do
+    not_a_page = "http://example.com/definitelynotapage"
+    expect(crawler.visit(not_a_page)).to be_nil
+  end
+
+  it "visits links breadth-first" do
+    allow_any_instance_of(Anaximander::Page).to receive(:open).and_return("")
+
+    root     = Anaximander::Page.new("http://example.com")
+    pricing  = Anaximander::Page.new("http://example.com/pricing")
+    features = Anaximander::Page.new("http://example.com/features")
+
+    allow(Anaximander::Page).to receive(:new).with("http://example.com/pricing").and_return(pricing)
+    allow(Anaximander::Page).to receive(:new).with("http://example.com/features").and_return(features)
+
+    allow(root).to receive_messages(links: ["http://example.com/pricing", "http://example.com/features"])
+    allow(pricing).to receive_messages(links: ["http://example.com/features"])
+
+    crawler.crawl(root)
+    expect(root.children).to eq [pricing, features]
+  end
+
+  it "crawls a multi-tiered website", :endtoend do
+    # See `spec/data/site` for the website hierarchy; it matches
+    # the directory structure.
+
+    crawler = described_class.new("http://localhost:#{@port}/index.html")
+    crawler.crawl
+
+    root = crawler.root
+
+    pricing = root.children[0]
+    features = root.children[1]
+
+    pricing_low = pricing.children[0]
+    pricing_med = pricing.children[1]
+    pricing_high = pricing.children[2]
+
+    sortof_high = pricing_high.children[0]
+    super_high = pricing_high.children[1]
+
+    expect(root.children.size).to eq 2
+    expect(root.children).to eq [pricing, features]
+
+    expect(pricing.children.size).to eq 3
+    expect(pricing.children).to eq [pricing_low, pricing_med, pricing_high]
+
+    expect(pricing_high.children.size).to eq 2
+    expect(pricing_high.children).to eq [sortof_high, super_high]
+
+    expect(features.children.size).to eq 0
+  end
+end

data/spec/anaximander/discovery/assets_spec.rb

@@ -0,0 +1,12 @@
+require "spec_helper"
+
+describe Anaximander::Discovery::Assets do
+  let(:html)   { Nokogiri::HTML(open("spec/data/page.html")) }
+  let(:assets) { described_class.new(html) }
+
+  # See `spec/data/page.html` for the HTML used in this test.
+
+  it "collects all CSS and Javascript assets" do
+    expect(assets.to_a).to eq(["main.css", "other.css", "allthethings.js"])
+  end
+end

data/spec/anaximander/discovery/links_spec.rb

@@ -0,0 +1,33 @@
+require "spec_helper"
+
+describe Anaximander::Discovery::Links do
+  let(:html)  { Nokogiri::HTML(open("spec/data/page.html")) }
+  let(:links) { described_class.new(html, "http://example.com/") }
+
+  # See `spec/data/page.html` for the HTML used in this test.
+
+  it "collects each unique link" do
+    expect(links.size).to eq(2)
+  end
+
+  it "only collects links within the same domain" do
+    expect(links).to_not include "http://example.net"
+  end
+
+  it "removes trailing slashes from URLs" do
+    expect(links).to_not include "http://example.com/"
+  end
+
+  it "disgards hash links" do
+    links.each { |link| expect(link).to_not include "#" }
+  end
+
+  it "expands relative paths to absolute paths" do
+    expect(links).to include "http://example.com/google"
+  end
+
+  it "expands relative paths to absolute paths using the base url of the page" do
+    links = described_class.new(html, "http://example.com/some/nested/page/")
+    expect(links).to_not include "http://example.com/some/nested/page/google"
+  end
+end

data/spec/anaximander/page_spec.rb

@@ -0,0 +1,24 @@
+require "spec_helper"
+
+describe Anaximander::Page do
+  let(:html) { File.read(File.expand_path("../../data/page.html", __FILE__)) }
+  let(:page) { described_class.new("http://example.com") }
+
+  before do
+    expect_any_instance_of(described_class).to receive(:open).and_return(html)
+  end
+
+  it "has unique links" do
+    expect(Anaximander::Discovery::Links).to receive(:new).with(an_instance_of(Nokogiri::HTML::Document), "http://example.com")
+    page.links
+  end
+
+  it "has assets" do
+    expect(Anaximander::Discovery::Assets).to receive(:new).with(an_instance_of(Nokogiri::HTML::Document))
+    page.assets
+  end
+
+  it "is comparable by URL" do
+    expect(page).to eq(page.clone)
+  end
+end

data/spec/anaximander/renderer_spec.rb

@@ -0,0 +1,64 @@
+require "spec_helper"
+
+describe Anaximander::Renderer do
+  include Fakeout::SpecHelpers
+
+  let(:child) { double(url: "http://example.com/foo", assets: ["/root.css", "/child.js"], children: []) }
+  let(:root)  { double(url: "http://example.com", assets: ["/root.css"], children: [child]) }
+
+  subject(:renderer) { described_class.new(color: false) }
+
+  before :all do
+    Fakeout.activate!
+  end
+
+  after :all do
+    Fakeout.deactivate!
+  end
+
+  it "draws a root node" do
+    renderer.draw(root)
+    expect(stdout).to include %(└── http://example.com ["/root.css"])
+  end
+
+  it "draws a child node whose parent is not a tail" do
+    renderer.draw_child(child, "", false)
+    expect(stdout).to include %(│   ├── http://example.com/foo ["/root.css", "/child.js"])
+  end
+
+  it "draws a child node whose parent is a tail" do
+    renderer.draw_child(child, "", true)
+    expect(stdout).to include %(    ├── http://example.com/foo ["/root.css", "/child.js"])
+  end
+
+  it "draws a tail node whose parent is not a tail node" do
+    renderer.draw_tail(child, "", false)
+    expect(stdout).to include %(│   └── http://example.com/foo ["/root.css", "/child.js"])
+  end
+
+  it "draws a tail node whose parent is also a tail" do
+    renderer.draw_tail(child, "", true)
+    expect(stdout).to include %(    └── http://example.com/foo ["/root.css", "/child.js"])
+  end
+
+  it "draws an entire tree", :endtoend do
+    tree = Anaximander::Crawler.new("http://localhost:#{@port}/index.html")
+    tree.crawl
+
+    domain = Anaximander::Url.new(tree.root.url).base
+
+    renderer = Anaximander::Renderer.new(color: false)
+    renderer.draw(tree.root)
+
+    expect(stdout).to eq <<-TREE
+└── #{domain}/index.html ["/main.css", "/application.js"]
+    ├── #{domain}/pricing.html ["/main.css", "/application.js"]
+    │   ├── #{domain}/pricing/low.html ["/main.css"]
+    │   ├── #{domain}/pricing/medium.html ["/main.css"]
+    │   └── #{domain}/pricing/high.html ["/main.css"]
+    │       ├── #{domain}/pricing/high/sortof_high.html ["/main.css"]
+    │       └── #{domain}/pricing/high/super_high.html ["/main.css"]
+    └── #{domain}/features.html ["/main.css", "/application.js"]
+    TREE
+  end
+end

data/spec/anaximander/url_spec.rb

@@ -0,0 +1,45 @@
+require "spec_helper"
+
+describe Anaximander::Url do
+  it "exposes the scheme and host as base" do
+    base = described_class.new("http://example.com/foo/bar?baz=1").base
+    expect(base).to eq "http://example.com"
+  end
+
+  it "joins other Urls" do
+    url = described_class.new("http://example.com").join(described_class.new("/foo"))
+    expect(url).to eq described_class.new("http://example.com/foo")
+  end
+
+  it "joins with a String url" do
+    url = described_class.new("http://example.com").join("/foo")
+    expect(url).to eq described_class.new("http://example.com/foo")
+  end
+
+  it "creates the absolute path when representing a relative path" do
+    url = described_class.new("/foo")
+    expect(url.absolute("http://example.com")).to eq described_class.new("http://example.com/foo")
+  end
+
+  it "does not create the absolute path if already representing one" do
+    url = described_class.new("http://example.com/foo")
+    expect(url.absolute("http://example.net")).to eq described_class.new("http://example.com/foo")
+  end
+
+  it "removes the fragment" do
+    url = described_class.new("http://example.com/#one-page-app")
+    expect(url.without_fragment).to eq described_class.new("http://example.com/")
+  end
+
+  it "is comparable to other Urls" do
+    url1 = described_class.new("http://example.com")
+    url2 = described_class.new("http://example.com")
+    expect(url1).to eq(url2)
+  end
+
+  it "is comparable to a string URL" do
+    url1 = described_class.new("http://example.com")
+    url2 = "http://example.com"
+    expect(url1 == url2).to eq true
+  end
+end

data/spec/data/page.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <link rel="stylesheet" href="main.css" type="text/css">
+  <link rel="stylesheet" href="other.css" type="text/css">
+  <script type="text/javascript" src="allthethings.js"></script>
+</head>
+<body>
+  <a href="http://example.com">Example</a>
+  <a href="/google">Google</a>
+  <blink>OMG A BLINK TAG</blink>
+  <a href="http://example.com">Duplicate Example</a>
+  <a href="http://example.net">Different Domain</a>
+  <a href="#">Some javascript thing</a>
+  <marquee>TODAY, IN NEWS!</marquee>
+</body>
+</html>

data/spec/data/site/config.ru

@@ -0,0 +1,6 @@
+use Rack::Static, root: "public"
+run lambda { |env|
+  page = env["PATH_INFO"].empty? ? "index.html" : env["PATH_INFO"]
+  file = File.open("public/#{page}", File::RDONLY)
+  [200, {"Content-Type" => "text/html"}, file]
+}

data/spec/data/site/public/features.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <script type="text/javascript" src="/application.js"></script>
+</body>
+</html>

data/spec/data/site/public/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <script type="text/javascript" src="/application.js"></script>
+</body>
+</html>

data/spec/data/site/public/pricing/high/sortof_high.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <h1>SORTOF HIGH</h1>
+</body>
+</html>
+

data/spec/data/site/public/pricing/high/super_high.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <h1>SUPER HIGH</h1>
+</body>
+</html>
+
+

data/spec/data/site/public/pricing/high.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <h1>High</h1>
+
+    <a href="/pricing/high/sortof_high.html">Sortof High</a>
+    <a href="/pricing/high/super_high.html">Super High</a>
+</body>
+</html>
+
+

data/spec/data/site/public/pricing/low.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <h1>Low</h1>
+</body>
+</html>
+
+

data/spec/data/site/public/pricing/medium.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <h1>Medium</h1>
+</body>
+</html>
+
+

data/spec/data/site/public/pricing.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <link rel="stylesheet" href="/main.css" type="text/css" media="screen" charset="utf-8">
+</head>
+<body>
+    <nav>
+        <a href="/pricing.html">Pricing</a>
+        <a href="/features.html">Features</a>
+    </nav>
+
+    <ul>
+        <li><a href="/pricing/low.html">$0</a>
+        <li><a href="/pricing/medium.html">$10</a>
+        <li><a href="/pricing/high.html">$20</a>
+    </ul>
+
+    <script type="text/javascript" src="/application.js"></script>
+</body>
+</html>
+

data/spec/spec_helper.rb

@@ -0,0 +1,40 @@
+require "socket"
+require "net/http"
+require "fakeout/safe"
+require "anaximander"
+
+RSpec.configure do |c|
+  c.before do
+    Anaximander.logger = false
+  end
+
+  c.before :each, :endtoend do
+    @port = obtain_port
+    @pid  = Process.spawn({}, "rackup -p #{@port}", chdir: File.expand_path("../data/site", __FILE__), out: "/dev/null", err: "/dev/null")
+    sleep 0.1 until server_running?
+  end
+
+  c.after :each, :endtoend do
+    Process.kill("INT", @pid)
+  end
+
+  # TCPServer, given a port of 0, will ask the OS for a
+  # random, available, port.
+  #
+  def obtain_port
+    server = TCPServer.new("127.0.0.1", 0)
+    port   = server.addr[1]
+    server.close
+    port
+  end
+
+  # Try to retrieve the homepage of the test site.
+  #
+  def server_running?
+    http = Net::HTTP.new("localhost", @port)
+    req  = Net::HTTP::Get.new("/index.html")
+    http.request(req).is_a?(Net::HTTPOK)
+  rescue Errno::ECONNREFUSED
+    false
+  end
+end

metadata

@@ -0,0 +1,208 @@
+--- !ruby/object:Gem::Specification
+name: anaximander
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Matte Noble
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-autotest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeout
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Web scraper that collects assets and links.
+email:
+- me@mattenoble.com
+executables:
+- mapgen
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- anaximander.gemspec
+- bin/mapgen
+- lib/anaximander.rb
+- lib/anaximander/cli.rb
+- lib/anaximander/crawler.rb
+- lib/anaximander/discovery/assets.rb
+- lib/anaximander/discovery/links.rb
+- lib/anaximander/page.rb
+- lib/anaximander/renderer.rb
+- lib/anaximander/url.rb
+- lib/anaximander/version.rb
+- spec/anaximander/crawler_spec.rb
+- spec/anaximander/discovery/assets_spec.rb
+- spec/anaximander/discovery/links_spec.rb
+- spec/anaximander/page_spec.rb
+- spec/anaximander/renderer_spec.rb
+- spec/anaximander/url_spec.rb
+- spec/data/page.html
+- spec/data/site/config.ru
+- spec/data/site/public/features.html
+- spec/data/site/public/index.html
+- spec/data/site/public/pricing.html
+- spec/data/site/public/pricing/high.html
+- spec/data/site/public/pricing/high/sortof_high.html
+- spec/data/site/public/pricing/high/super_high.html
+- spec/data/site/public/pricing/low.html
+- spec/data/site/public/pricing/medium.html
+- spec/spec_helper.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Web scraper that collects assets and links.
+test_files:
+- spec/anaximander/crawler_spec.rb
+- spec/anaximander/discovery/assets_spec.rb
+- spec/anaximander/discovery/links_spec.rb
+- spec/anaximander/page_spec.rb
+- spec/anaximander/renderer_spec.rb
+- spec/anaximander/url_spec.rb
+- spec/data/page.html
+- spec/data/site/config.ru
+- spec/data/site/public/features.html
+- spec/data/site/public/index.html
+- spec/data/site/public/pricing.html
+- spec/data/site/public/pricing/high.html
+- spec/data/site/public/pricing/high/sortof_high.html
+- spec/data/site/public/pricing/high/super_high.html
+- spec/data/site/public/pricing/low.html
+- spec/data/site/public/pricing/medium.html
+- spec/spec_helper.rb