crawl 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source 'http://rubygems.org'
+
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/crawl

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require_relative '../lib/crawl.rb'
+
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = "Exhaustive search pages witin a domain, reporting any page that returns a bad response code\nUsage: crawl [options] domain"
+  opts.on('-s', '--start /home,/about', Array, 'Starting path(s), defaults to /') { |o| options[:start] = o }
+  opts.on('-u', '--username username', String, 'Basic auth username') { |o| options[:username] = o }
+  opts.on('-p', '--password password', String, 'Basic auth password') { |o| options[:password] = o }
+  opts.on('-c', '--ci', 'Output files for CI integration') { |o| options[:ci] = o }
+  opts.on('-v', '--verbose', 'Give details when crawling') { |o| options[:verbose] = o }
+  opts.on_tail("-h", "--help", "Show this message") { |o| puts opts; exit }
+end.parse!
+
+options.merge!(domain: optparse.first)
+
+unless options[:domain]
+  puts 'Must provide a domain'
+  exit -1
+end
+
+crawler = Crawl::Engine.new(options)
+
+trap("SIGINT") do
+    puts "\n\nAborting crawl.."
+    crawler.summarize
+    exit -1
+end
+
+crawler.run
+crawler.summarize
+
+unless crawler.errors.empty?
+  puts 'Errors during crawling'
+  exit -1
+end

data/crawl.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/crawl/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Tor Erik Linnerud"]
+  gem.email         = ["tor@alphasights.com"]
+  gem.description   = "Crawl all pages on a domain, checking for errors"
+  gem.summary       = "Exhaustive search pages witin a domain, reporting any page that returns a bad response code"
+  gem.homepage      = "http://github.com/alphasights/crawl"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "crawl"
+  gem.require_paths = ["lib"]
+  gem.version       = Crawl::VERSION
+  gem.add_dependency('nokogiri')
+  gem.add_dependency('rest-client')
+  gem.add_dependency('ci_reporter')
+end

data/lib/crawl/engine.rb

@@ -0,0 +1,167 @@
+# encoding: utf-8
+class Crawl::Engine
+  DEFAULT_OPTIONS = {:domain => '',
+                     :start => ['/'],
+                     :username => '',
+                     :password => '',
+                     :verbose => false,
+                     :session_id => false}
+
+
+  IGNORE = [/#/, /mailto:/, /skype:/, /logout/, /javascript:/, %r(/xhr/), /https:/, /\.pdf$/, /^$/]
+  VALID_RESPONSE_CODES = [200, 302]
+  MAX_REDIRECTS = 3
+  LINE_WIDTH = 78
+
+  Result = Struct.new(:url, :object)
+
+  attr_reader :options, :errors
+
+
+  def initialize(caller_options = {})
+    @options = DEFAULT_OPTIONS.merge(caller_options)
+    @authorization = Base64.encode64("#{options[:username]}:#{options[:password]}")
+
+    @found_links = options[:start].to_set
+    @link_sources = {}
+    @found_links.each {|target| @link_sources[target] = 'Initial'}
+    @visited_links = Set[]
+    @visited_documents = Set[]
+    @invalid_links = Set[]
+    @broken_pages = []
+    @errors = []
+    @verbose = options[:verbose] || ENV['VERBOSE']
+    @number_of_dots = 0
+    @report_manager = CI::Reporter::ReportManager.new("crawler") if options[:ci]
+  end
+
+  def run
+    until (links = @found_links - (@visited_links + @invalid_links)).empty? do
+      links.each do |link|
+        puts "\nChecking #{link}" if @verbose
+        next unless response = retrieve(link)
+        next unless response.headers[:content_type] =~ %r{text/html}
+        @visited_documents << link
+        @found_links += links = find_links(link, response.to_str)
+        # validate(link, response.body_str)
+      end
+    end
+  end
+
+
+
+  def summarize
+    if @errors.size > 0
+
+      @errors.each do |error|
+        puts "\n#{error.url}"
+        puts "  Linked from #{linked_from(error.url)}"
+        puts error.object.to_s.word_wrap.split("\n").map{|line| '  ' + line}
+      end
+
+      print(<<-SUM)
+
+Pages crawled: #{@visited_documents.size}
+Pages with errors: #{@errors.size - @invalid_links.size}
+Broken pages: #{@broken_pages.size}
+Invalid links: #{@invalid_links.size}
+
+I=Invalid P=Parse Error S=Status code bad
+
+SUM
+      exit(@errors.size)
+    else
+       puts "\n\n#{@visited_documents.size} pages crawled"
+    end
+
+    puts
+  end
+
+private
+
+  def validate(link, body)
+    puts "  Validating..." if @verbose
+
+    json_response = RestClient.post 'http://validator.nu?out=json', body, :content_type => 'text/html; charset=utf-8'
+    messages = JSON.parse(json_response.body)['messages']
+    error_messages = messages.select { |message| message['type'] != 'info' }
+
+    if error_messages.empty?
+      handle_success
+      true
+    else
+      response = error_messages.map do |message|
+        type, message = message['type'], message['message']
+        type_color = type == 'error' ? 31 : 33
+        "\e[#{type_color};1m" + type.capitalize + "\e[0m: " + message
+      end.join("\n\n")
+
+      @errors << Result.new(link, response)
+      handle_error('I')
+      false
+    end
+  rescue RestClient::ServiceUnavailable
+    handle_error('U')
+    false
+  end
+
+  def retrieve(link)
+    test_suite = CI::Reporter::TestSuite.new(link)
+    test_case  = CI::Reporter::TestCase.new(link)
+    test_suite.start
+    test_case.start
+    puts "  Fetching.." if @verbose
+
+    headers = {}
+    #headers.merge!(Authorization: "Basic #{@authorization}") if options[:username]
+    headers.merge(user: options[:username], password: options[:password])
+    response = RestClient.get(options[:domain] + link, headers)
+    test_suite.name = link
+    test_case.name = link
+    test_case.finish
+    @visited_links << link
+    unless VALID_RESPONSE_CODES.include?(response.code)
+      @errors << Result.new(link, "Status code was #{response.code}")
+      @broken_pages << link
+      test_case.failures << Crawl::Failure.new(link, response.code, linked_from(link))
+      test_suite.testcases << test_case
+      test_suite.finish
+      @report_manager.write_report(test_suite) if options[:ci]
+      return nil
+    end
+    test_suite.testcases << test_case
+    test_suite.finish
+    @report_manager.write_report(test_suite) if options[:ci]
+    return response
+  rescue RestClient::InternalServerError => e
+    @errors << Result.new(link, "Error whilst retrieving page: #{e.message}")
+    @invalid_links << link
+    return nil
+  end
+
+  def linked_from(target)
+    @link_sources[target] # => source
+  end
+
+  def find_links(source_link, body)
+    puts "  Finding links.." if @verbose
+    doc = Nokogiri::HTML(body)
+    anchors = doc.css('a').to_a
+    anchors.reject!{|anchor| anchor['onclick'].to_s =~ /f.method = 'POST'/}
+    anchors.reject!{|anchor| anchor['data-method'] =~ /put|post|delete/ }
+    anchors.reject!{|anchor| anchor['class'].to_s =~ /unobtrusive_/}
+    raw_links = anchors.map{|anchor| anchor['href']}
+    raw_links.compact!
+    raw_links.map!{|link| link.sub(options[:domain], '')}
+    raw_links.delete_if{|link| link =~ %r{^http://}}
+    raw_links.delete_if{|link| IGNORE.any?{|pattern| link =~ pattern}}
+    raw_links.each do |target_link|
+      unless @found_links.include?(target_link)
+        puts "    Adding #{target_link} found on #{source_link}" if @verbose
+        @link_sources[target_link] = source_link
+      end
+    end
+
+    raw_links
+  end
+end

data/lib/crawl/failure.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+class Crawl::Failure
+  attr_reader :link, :code, :from
+
+  def initialize(link, code, from)
+    @link = link
+    @code = code
+    @from = from
+  end
+
+  def failure?
+    true
+  end
+
+  def error?
+    !failure?
+  end
+
+  def name
+    link
+  end
+
+  def message
+    "Status code was #{code}"
+  end
+
+  def location
+    "Linked from #{from}"
+  end
+end

data/lib/crawl/string.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+class String
+  def word_wrap(line_width = 80)
+    self.split("\n").collect do |line|
+      line.length > line_width ? line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip : line
+    end * "\n"
+  end
+end

data/lib/crawl/version.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+module Crawl
+  VERSION = "0.0.1"
+end

data/lib/crawl.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+puts require('nokogiri')
+puts require('rest_client')
+require 'ci/reporter/core'
+
+require 'base64'
+require 'set'
+require 'fileutils'
+require 'digest/sha1'
+require 'json'
+require 'tempfile'
+require 'tmpdir'
+
+require_relative "crawl/version"
+require_relative "crawl/engine"
+require_relative "crawl/string"
+require_relative "crawl/failure"

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: crawl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Tor Erik Linnerud
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-11-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70363418401240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70363418401240
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: &70363418400700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70363418400700
+- !ruby/object:Gem::Dependency
+  name: ci_reporter
+  requirement: &70363418400280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70363418400280
+description: Crawl all pages on a domain, checking for errors
+email:
+- tor@alphasights.com
+executables:
+- crawl
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- bin/crawl
+- crawl.gemspec
+- lib/crawl.rb
+- lib/crawl/engine.rb
+- lib/crawl/failure.rb
+- lib/crawl/string.rb
+- lib/crawl/version.rb
+homepage: http://github.com/alphasights/crawl
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Exhaustive search pages witin a domain, reporting any page that returns a
+  bad response code
+test_files: []