shelob 0.1.0.beta1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4fd39cfeaa059a074821a0d60ee93b464ff48819
+  data.tar.gz: fc04fd19c13c1a970abfed9c126854da9ad6eaeb
+SHA512:
+  metadata.gz: 72b0645723887dfb1942108c93e5cfda5f50de550d5a240f5da0119763fbd02745fdce0f88038b42849b863e54a81cf214db6285badab74eed6dbb082debd8ab
+  data.tar.gz: e53ddc74da61a78b19acde7d9e87859c9682dc48bd044f736f51942d1ee51d79b573b868ee5c72e3c87e9ab1e4e922f6c3760357947a2497492f1b1ce9ecd693

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in link_checker.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,7 @@
+guard 'minitest' do
+ 
+  # with Minitest::Unit
+  watch(%r|^test/(.*)\/?test_(.*)\.rb$|)
+  watch(%r|^lib/(.*?)([^/]+)\.rb$|)     { |m| "test/#{m[1]}test_#{m[2]}.rb" }
+ 
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Benjamin Nicholas
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# LinkChecker
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'link_checker'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install link_checker
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/test_*.rb"
+end
+
+task :default => 'test'

data/bin/shelob

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'shelob'
+
+def main args
+  puts Shelob::Spider.new(args[0], verbose: options[:verbose]).check
+end
+
+options = {verbose: 0}
+optparse = OptionParser.new do |opts|
+  opts.banner = "Usage: shelob [options] root_url"
+
+  opts.on('-v', "--[no-]verbose", "Print simple information(overrides -r)") do
+    options[:verbose] = 1
+  end
+
+  opts.on('-r', '--[no-]really-verbose', "Print lots of information(overrides -v)") do
+    options[:verbose] = 2
+  end
+  
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+if ARGV.empty?
+  puts optparse
+  exit 1
+end
+
+begin
+  exit main(ARGV)
+rescue => ex
+  STDERR.puts ex.message
+end
+

data/lib/extractor.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'nokogiri'
+
+module Shelob
+  class Extractor
+    def initialize fetched
+      @fetched = fetched
+    end
+
+    def extract
+      content = Nokogiri::HTML(@fetched.body)
+      raw = content.css('a').map { |anchor| anchor['href'] }
+      raw.map do |link| 
+        if link.start_with? '/' 
+          u = URI(@fetched.url)
+          "#{u.scheme}://#{u.host}#{link}"
+        else
+          link
+        end
+      end
+    end
+  end
+end

data/lib/link_result.rb

@@ -0,0 +1,13 @@
+class LinkResult
+  attr_reader :url, :status, :body
+
+  def initialize url, status, body
+    @url = url
+    @status = status
+    @body = body
+  end
+
+  def to_s
+    "#{@status}: #{@url}"
+  end
+end

data/lib/resolver.rb

@@ -0,0 +1,16 @@
+require 'link_result'
+require 'net/http'
+
+module Shelob
+  class Resolver
+    def initialize url
+      @uri = URI(url)
+    end
+
+    def resolve
+      resp = Net::HTTP.get_response(@uri)
+
+      LinkResult.new @uri.to_s, resp.code.to_i, resp.body
+    end
+  end
+end

data/lib/shelob/version.rb

@@ -0,0 +1,3 @@
+module Shelob
+  VERSION = "0.1.0.beta1"
+end

data/lib/shelob.rb

@@ -0,0 +1,64 @@
+require "shelob/version"
+require "resolver"
+require "extractor"
+require "set"
+
+module Shelob
+  class Spider
+    attr_accessor :hostname
+
+    def initialize hostname, options = {}
+      @hostname = hostname
+      @queue = [ hostname ]
+      @urls = Set.new @queue
+      @failures = []
+      @verbose = options[:verbose] == 1 ? true : false
+      @chatty = options[:verbose] == 2 ? true : false
+    end
+
+    def check
+      while not @queue.empty?
+        url = @queue.shift
+        @urls << url
+
+        if @verbose
+          print '.'
+        end
+
+        if @chatty
+          print "#{url}... "
+        end
+
+        fetch = Resolver.new(url).resolve
+
+        @failures << fetch if fetch.status >= 400
+
+        links = Extractor.new(fetch).extract
+
+        filtered = links.select do |link| 
+          link.start_with? @hostname and !@urls.include? link
+        end
+
+        if @chatty
+          puts "checked!"
+        end
+
+        @queue.push(*filtered)
+      end
+
+      @failures
+    end
+
+    def remaining
+      return @queue.count
+    end
+
+    def requests
+      return @urls.count
+    end
+    
+    def fetched
+      return @urls
+    end
+  end
+end

data/shelob.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'shelob/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "shelob"
+  spec.version       = Shelob::VERSION
+  spec.authors       = ["Benjamin Nicholas"]
+  spec.email         = ["bnicholas@brandnetworksinc.com"]
+  spec.description   = %q{A giant spider that starts on a given page, finds all links on the page, ensure they resolve, and recurses if the link is underneath the starting url}
+  spec.summary       = %q{Spider a site and check links}
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-minitest"
+
+  spec.add_runtime_dependency     "nokogiri"
+end

data/test/test_extractor.rb

@@ -0,0 +1,37 @@
+require 'minitest/autorun'
+require 'extractor'
+require 'link_result'
+
+describe Shelob::Extractor, "Link extracting module" do
+
+  describe "when created" do
+    it "should be created with a LinkResult" do
+      le = LinkResult.new("http://google.com", 200, '<html><head><title>resume</title></head><body><a href="http://bmnick.com">home</a><a href="http://bmnick.com/resume/resume.pdf">pdf</a></body></html>')
+      le.wont_be_nil
+    end
+  end
+
+  describe "when used" do
+    before do
+      @result = LinkResult.new("http://google.com", 200, '<html><head><title>hi</title></head><body><a href="http://bing.com">bing</a><a href="http://yahoo.com">yahoo</a></body></html>')
+      @result2 = LinkResult.new("http://google.com/something", 200, '<html><head><title>hi</title></head><body><a href="/about">about</a></body></html>')
+      @le = Shelob::Extractor.new(@result)
+      @le2 = Shelob::Extractor.new(@result2)
+    end
+
+    it "should return a list of the links in the page" do
+      extracts = @le.extract 
+      extracts.must_be_kind_of Array
+      extracts.must_equal ["http://bing.com", "http://yahoo.com"]
+    end
+
+    it "should transform relative links to absolute" do
+      extracts = @le2.extract
+      extracts.must_be_kind_of Array
+      extracts.must_equal ["http://google.com/about"]
+    end
+
+  end # describe
+
+end # describe
+

data/test/test_link_result.rb

@@ -0,0 +1,29 @@
+require 'minitest/autorun'
+require 'link_result'
+
+describe LinkResult, "Link fetch result" do
+  before do
+    @result = LinkResult.new("http://google.com", 200, '<html><head><title>hi</title></head><body><a href="http://bing.com">bing</a><a href="http://yahoo.com">yahoo</a></body></html>')
+  end
+
+  describe "when created" do
+    it "should take three arguments" do
+      @result.wont_be_nil
+    end
+
+    it "should save arguments" do
+      @result.url.must_equal "http://google.com"
+      @result.status.must_equal 200
+      @result.body.must_equal '<html><head><title>hi</title></head><body><a href="http://bing.com">bing</a><a href="http://yahoo.com">yahoo</a></body></html>'
+    end
+
+    it "should be immutable" do
+      proc { @result.status = 404 }.must_raise NoMethodError
+    end
+
+    it "should have a clean string rep" do
+      @result.to_s.must_equal "200: http://google.com"
+    end
+  end
+end
+

data/test/test_resolver.rb

@@ -0,0 +1,31 @@
+require 'minitest/autorun'
+require 'webmock/minitest'
+require 'resolver'
+require 'link_result'
+
+WebMock.allow_net_connect!
+
+describe Shelob::Resolver, "Link fetching module" do 
+  describe "when created" do
+    it "should be created with a url" do
+      Shelob::Resolver.new("http://bmnick.com/ruby-c-extensions") 
+    end
+  end
+
+  describe "when used" do
+    before do
+      @resolver = Shelob::Resolver.new("http://bmnick.com/ruby-c-extensions")
+      @result = @resolver.resolve
+    end
+
+    it "should return a LinkResult" do
+      @result.must_be_kind_of LinkResult
+    end
+
+    it "should return live result" do
+      @result.body.must_match(/CExt/)
+    end
+
+  end
+      
+end

data/test/test_shelob.rb

@@ -0,0 +1,97 @@
+require 'minitest/autorun'
+require 'webmock/minitest'
+require 'shelob'
+
+# Stub out requests
+
+describe Shelob, "Link checking module" do
+  describe "when created" do
+    it "should exist" do
+      Shelob.wont_be_nil
+    end
+  end
+end
+
+describe Shelob::Spider, "Link checking spider" do
+  before do
+    stub_request(:any, 'http://bmnick.com/resume').to_return(body: '<html><head><title>resume</title></head><body><a href="http://bmnick.com">home</a><a href="http://bmnick.com/resume/resume.pdf">pdf</a><a href="http://bmnick.com/resume/secret"</body></html>')
+    stub_request(:any, 'http://bmnick.com/').to_return(status: 200, body: '<html><head><title>pdf</title></head><body><a href="http://bmnick.com/resume/">resume</a><a href="http://bmnick.com/">home</a><a href="http://bmnick.com/resume/secret">no touchy!</a></body></html>')
+    stub_request(:any, 'http://bmnick.com/resume/secret').to_return(body: '<html><head><title>secrets</title></head><body><a href="http://bmnick.com/resume/boring">boredom</a><a href="http://bmnick.com/resume">resume</a><a href="/resume/relative">relative</a></body></html>"')
+    stub_request(:any, 'http://bmnick.com/resume/resume.pdf').to_return(status: 404)
+    stub_request(:any, 'http://bmnick.com/resume/boring').to_return(status: 500)
+    stub_request(:any, 'http://bmnick.com/resume/relative').to_return(status: 204)
+  end
+  describe "when created" do
+    it "should exist" do
+      Shelob::Spider.wont_be_nil
+    end
+    it "should store the initial url" do
+      spider = Shelob::Spider.new("https://openforum.com")
+      spider.wont_be_nil
+      spider.hostname.must_equal "https://openforum.com"
+    end
+  end
+  describe "when checking links" do
+    before do
+
+      @spider = Shelob::Spider.new("http://bmnick.com/resume")
+      @results = @spider.check
+    end
+
+    it "should return an array from check" do 
+      @results.must_be_kind_of Array
+    end
+    it "should return only error links" do
+      @results.select{|r| r.status == 200}.must_be_empty
+    end
+    it "should provide remaining counts" do
+      @spider.remaining.must_equal 0
+    end
+    it "should fetch the original url" do
+      @spider.fetched.must_include "http://bmnick.com/resume"
+    end
+    it "should provide a number of urls fetched" do
+      # http://bmnick.com/resume
+      # http://bmnick.com/resume/resume.pdf
+      # http://bmnick.com/resume/secret
+      # http://bmnick.com/resume/boring
+      # http://bmnick.com/resume/relative
+      @spider.requests.must_equal 5
+    end
+    it "should make a web request for the original url" do
+      assert_requested :get, "http://bmnick.com/resume"
+    end
+    it "should make a web request for child urls" do
+      # 404
+      assert_requested :get, "http://bmnick.com/resume/resume.pdf"
+      @spider.fetched.must_include "http://bmnick.com/resume/resume.pdf"
+
+      # successful
+      assert_requested :get, "http://bmnick.com/resume/secret"
+      @spider.fetched.must_include "http://bmnick.com/resume/secret"
+    end
+    it "should return the failed request" do
+      # http://bmnick.com/resume/resume.pdf => 404
+      # http://bmnick.com/resume/boring => 500
+      @results.count.must_equal 2
+    end
+    it "shouldn't request pages without the prefix" do
+      assert_not_requested :get, "http://bmnick.com"
+    end
+    it "shouldn't request pages multiple times" do
+      assert_requested :get, "http://bmnick.com/resume", times: 1
+    end
+    it "should continue to spider down the page" do
+      assert_requested :get, "http://bmnick.com/resume/boring"
+      @spider.fetched.must_include "http://bmnick.com/resume/boring"
+    end
+    it "should support relative links" do
+      assert_requested :get, "http://bmnick.com/resume/relative"
+      @spider.fetched.must_include "http://bmnick.com/resume/relative"
+    end
+    it "should format a string cleanly" do
+      @results.map{|r|r.to_s}.join("\n").must_equal "404: http://bmnick.com/resume/resume.pdf
+500: http://bmnick.com/resume/boring"
+    end
+  end
+end

metadata

@@ -0,0 +1,165 @@
+--- !ruby/object:Gem::Specification
+name: shelob
+version: !ruby/object:Gem::Version
+  version: 0.1.0.beta1
+platform: ruby
+authors:
+- Benjamin Nicholas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A giant spider that starts on a given page, finds all links on the page,
+  ensure they resolve, and recurses if the link is underneath the starting url
+email:
+- bnicholas@brandnetworksinc.com
+executables:
+- shelob
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/shelob
+- lib/extractor.rb
+- lib/link_result.rb
+- lib/resolver.rb
+- lib/shelob.rb
+- lib/shelob/version.rb
+- shelob.gemspec
+- test/test_extractor.rb
+- test/test_link_result.rb
+- test/test_resolver.rb
+- test/test_shelob.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Spider a site and check links
+test_files:
+- test/test_extractor.rb
+- test/test_link_result.rb
+- test/test_resolver.rb
+- test/test_shelob.rb