crawl 0.0.5 → 0.1.0.beta1

data/bin/crawl

@@ -32,7 +32,7 @@ end
 crawler.run
 crawler.summarize
 
-unless crawler.errors.empty?
+if crawler.errors?
   puts 'Errors during crawling'
   exit -1
 end

data/crawl.gemspec

@@ -17,4 +17,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency('nokogiri')
   gem.add_dependency('rest-client')
   gem.add_dependency('ci_reporter')
+  gem.add_dependency('eventmachine', '~> 1.0.0.beta.4')
+  gem.add_dependency('em-http-request')
 end

data/lib/crawl/engine.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+
 class Crawl::Engine
   DEFAULT_OPTIONS = {:domain => '',
                      :start => ['/'],
@@ -13,69 +14,42 @@ class Crawl::Engine
   MAX_REDIRECTS = 3
   LINE_WIDTH = 78
 
-  Result = Struct.new(:url, :object)
-
-  attr_reader :options, :errors
-
+  attr_reader :options
 
   def initialize(caller_options = {})
     @options = DEFAULT_OPTIONS.merge(caller_options)
     @authorization = Base64.encode64("#{options[:username]}:#{options[:password]}")
-
-    @found_links = options[:start].to_set
-    @link_sources = {}
-    @found_links.each {|target| @link_sources[target] = 'Initial'}
-    @visited_links = Set[]
-    @visited_documents = Set[]
-    @invalid_links = Set[]
-    @broken_pages = []
-    @errors = []
     @verbose = options[:verbose] || ENV['VERBOSE']
-    @number_of_dots = 0
-    @report_manager = CI::Reporter::ReportManager.new("crawler") if options[:ci]
     @validate_markup = options[:markup]
+    @register = Crawl::Register.new(options[:start].to_a)
+
+    @report_manager = CI::Reporter::ReportManager.new("crawler") if options[:ci]
   end
 
   def run
-    until (links = @found_links - (@visited_links + @invalid_links)).empty? do
-      links.each do |link|
-        puts "\nChecking #{link}" if @verbose
-        next unless response = retrieve(link)
-        next unless response.headers[:content_type] =~ %r{text/html}
-        @visited_documents << link
-        @found_links += links = find_links(link, response.to_str)
-        validate(link, response.body) if @validate_markup
-      end
+    EventMachine.run do
+      process_next
     end
   end
 
-
+  def process_next
+    return if @register.processing_size >= EM.threadpool_size
+    if @register.finished?
+      EventMachine.stop
+    elsif (link = @register.next_link)
+      puts "\nChecking #{link}" if @verbose
+      retrieve(link)
+      # validate(link, response.body) if @validate_markup
+      process_next
+    end
+  end
 
   def summarize
-    if @errors.size > 0
-
-      @errors.each do |error|
-        puts "\n#{error.url}"
-        puts "  Linked from #{linked_from(error.url)}"
-        puts error.object.to_s.word_wrap.split("\n").map{|line| '  ' + line}
-      end
-
-      print(<<-SUM)
-
-Pages crawled: #{@visited_documents.size}
-Pages with errors: #{@errors.size - @invalid_links.size}
-Broken pages: #{@broken_pages.size}
-Invalid links: #{@invalid_links.size}
-
-I=Invalid P=Parse Error S=Status code bad
-
-SUM
-      exit(@errors.size)
-    else
-       puts "\n\n#{@visited_documents.size} pages crawled"
-    end
+    @register.summarize
+  end
 
-    puts
+  def errors?
+    @register.errors?
   end
 
 private
@@ -96,7 +70,7 @@ private
         "\e[#{type_color};1m" + type.capitalize + "\e[0m: " + message
       end.join("\n\n")
 
-      @errors << Result.new(link, response)
+      @register.error link, response
       false
     end
   rescue RestClient::ServiceUnavailable
@@ -104,41 +78,72 @@ private
     false
   end
 
+  def register_error(link, message)
+    @register.error link, message
+    @register.returned_invalid link
+    process_next
+  end
+
   def retrieve(link)
-    test_suite = CI::Reporter::TestSuite.new(link)
-    test_case  = CI::Reporter::TestCase.new(link)
-    test_suite.start
-    test_case.start
-    puts "  Fetching.." if @verbose
-
-    attributes = {:method => :get, :url => options[:domain] + link}
-    attributes.merge!(user: options[:username], password: options[:password])
-    response = RestClient::Request.execute(attributes)
-    test_suite.name = link
-    test_case.name = link
-    test_case.finish
-    @visited_links << link
-    unless VALID_RESPONSE_CODES.include?(response.code)
-      @errors << Result.new(link, "Status code was #{response.code}")
-      @broken_pages << link
-      test_case.failures << Crawl::Failure.new(link, response.code, linked_from(link))
-      test_suite.testcases << test_case
-      test_suite.finish
-      @report_manager.write_report(test_suite) if options[:ci]
+    # test_suite = CI::Reporter::TestSuite.new(link)
+    # test_case  = CI::Reporter::TestCase.new(link)
+    # test_suite.start
+    # test_case.start
+    # test_suite.name = link
+    # test_case.name = link
+
+    puts "Fetching #{options[:domain] + link} ..." if @verbose
+
+    unless link.start_with? '/'
+      register_error(link, "Relative path found. Crawl does not support relative paths.")
       return nil
     end
-    test_suite.testcases << test_case
-    test_suite.finish
-    @report_manager.write_report(test_suite) if options[:ci]
-    return response
-  rescue RestClient::InternalServerError, RestClient::ResourceNotFound, RestClient::Unauthorized => e
-    @errors << Result.new(link, "Error whilst retrieving page: #{e.message}")
-    @invalid_links << link
-    return nil
+
+    http = EventMachine::HttpRequest.new(options[:domain] + link)
+    req = http.get :redirects => MAX_REDIRECTS, :head => {'authorization' => [options[:username], options[:password]]}
+    req.timeout(30)
+
+    req.errback do
+      if req.nil?
+         @register.retry(link, 'WAT?')
+         process_next
+       elsif msg = req.error
+         register_error(link, msg)
+       elsif req.response.nil? || req.response.empty?
+         # no response at all?
+         @register.retry(link, 'Timeout?')
+         # register_error(link, 'Timeout?')
+       else
+         @register.retry(link, 'Partial response: Server Broke Connection?')
+         process_next
+       end
+    end
+
+    req.callback do
+      if VALID_RESPONSE_CODES.include?(req.response_header.status)
+        @register.returned link
+        if req.response_header["CONTENT_TYPE"] =~ %r{text/html}
+          @register.add find_links(link, req.response.to_str)
+        end
+      else
+        @register.error link, "Status code was #{req.response_header.status}"
+        @register.returned_broken link
+        # test_case.failures << Crawl::Failure.new(link, req.response_header.status, linked_from(link))
+        # test_suite.testcases << test_case
+        # test_suite.finish
+        # @report_manager.write_report(test_suite) if options[:ci]
+      end
+      process_next
+    end
+
+    # test_case.finish
+    # test_suite.testcases << test_case
+    # test_suite.finish
+    # @report_manager.write_report(test_suite) if options[:ci]
   end
 
   def linked_from(target)
-    @link_sources[target] # => source
+    @register.source_for target
   end
 
   def find_links(source_link, body)
@@ -147,17 +152,17 @@ private
     anchors = doc.css('a').to_a
     anchors.reject!{|anchor| anchor['onclick'].to_s =~ /f.method = 'POST'/}
     anchors.reject!{|anchor| anchor['data-method'] =~ /put|post|delete/ }
+    anchors.reject!{|anchor| anchor['data-remote'] =~ /true/ }
     anchors.reject!{|anchor| anchor['class'].to_s =~ /unobtrusive_/}
+    anchors.reject!{|anchor| anchor['rel'].to_s =~ /nofollow/}
     raw_links = anchors.map{|anchor| anchor['href']}
     raw_links.compact!
     raw_links.map!{|link| link.sub(options[:domain], '')}
-    raw_links.delete_if{|link| link =~ %r{^http://}}
+    raw_links.delete_if{|link| link =~ %r{^http(s)?://}}
     raw_links.delete_if{|link| IGNORE.any?{|pattern| link =~ pattern}}
     raw_links.each do |target_link|
-      unless @found_links.include?(target_link)
-        puts "    Adding #{target_link} found on #{source_link}" if @verbose
-        @link_sources[target_link] = source_link
-      end
+      puts "    Adding #{target_link} found on #{source_link}" if @verbose
+      @register.set_link_source(target_link, source_link)
     end
 
     raw_links

data/lib/crawl/register.rb

@@ -0,0 +1,102 @@
+class Crawl::Register
+
+  Result = Struct.new(:url, :object)
+
+  def initialize(unprocessed)
+    @unprocessed = unprocessed
+    @processing = []
+    @processed = []
+
+    @invalid_links = Set[]
+    @broken_pages = Set[]
+
+    @errors = []
+    @link_sources = {}
+  end
+
+  def add(links)
+    new_links = links - @processed - @processing - @unprocessed
+    @unprocessed += new_links
+  end
+
+  def next_link
+    link = @unprocessed.shift
+    @processing << link if link
+    if @processing.size > EM.threadpool_size
+      puts "WARNING: #{@processing.size} pages are being process when EM threadpool only has #{EM.threadpool_size} threads."
+    end
+    link
+  end
+
+  def set_link_source(link, source)
+    @link_sources[link] = source
+  end
+
+  def source_for(link)
+    @link_sources.fetch link, '?'
+  end
+
+  def error(link, object)
+    @errors << Result.new(link, object)
+  end
+
+  def returned_invalid(link)
+    returned link
+    @invalid_links << link
+  end
+
+  def returned_broken(link)
+    returned link
+    @broken_pages << link
+  end
+
+  def returned(link)
+    @processed << link
+    @processing -= [link]
+  end
+
+  def finished?
+    @unprocessed.size + @processing.size == 0
+  end
+
+  def processing_size
+    @processing.size
+  end
+
+  def retry(link, reason)
+    puts "Retrying #{link} : #{reason}"
+    @processing -= [link]
+    @unprocessed << link
+  end
+
+  def summarize
+    if @errors.size > 0
+
+      @errors.each do |error|
+        puts "\n#{error.url}"
+        puts "  Linked from #{source_for error.url}"
+        puts error.object.to_s.word_wrap.split("\n").map{|line| '  ' + line}
+      end
+
+      print(<<-SUM)
+
+Pages crawled: #{@processed.size}
+Pages with errors: #{@errors.size - @invalid_links.size}
+Broken pages: #{@broken_pages.size}
+Invalid links: #{@invalid_links.size}
+
+I=Invalid P=Parse Error S=Status code bad
+
+SUM
+      exit(@errors.size)
+    else
+       puts "\n\n#{@processed.size} pages crawled"
+    end
+
+    puts
+  end
+
+  def errors?
+    @errors.size > 0
+  end
+end

data/lib/crawl/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Crawl
-  VERSION = "0.0.5"
+  VERSION = "0.1.0.beta1"
 end

data/lib/crawl.rb

@@ -2,7 +2,8 @@
 require('nokogiri')
 require('rest_client')
 require 'ci/reporter/core'
-
+require 'eventmachine'
+require 'em-http-request'
 require 'base64'
 require 'set'
 require 'fileutils'
@@ -14,4 +15,5 @@ require 'tmpdir'
 require_relative "crawl/version"
 require_relative "crawl/engine"
 require_relative "crawl/string"
-require_relative "crawl/failure"
+require_relative "crawl/failure"
+require_relative "crawl/register"

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: crawl
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease: 
+  version: 0.1.0.beta1
+  prerelease: 6
 platform: ruby
 authors:
 - Tor Erik Linnerud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-21 00:00:00.000000000 Z
+date: 2012-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70216317741600 !ruby/object:Gem::Requirement
+  requirement: &70366743805820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70216317741600
+  version_requirements: *70366743805820
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &70216317740600 !ruby/object:Gem::Requirement
+  requirement: &70366743804960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70216317740600
+  version_requirements: *70366743804960
 - !ruby/object:Gem::Dependency
   name: ci_reporter
-  requirement: &70216317739980 !ruby/object:Gem::Requirement
+  requirement: &70366750375000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,29 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70216317739980
+  version_requirements: *70366750375000
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &70366750374440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta.4
+  type: :runtime
+  prerelease: false
+  version_requirements: *70366750374440
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &70366750373840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70366750373840
 description: Crawl all pages on a domain, checking for errors
 email:
 - tor@alphasights.com
@@ -61,6 +83,7 @@ files:
 - lib/crawl.rb
 - lib/crawl/engine.rb
 - lib/crawl/failure.rb
+- lib/crawl/register.rb
 - lib/crawl/string.rb
 - lib/crawl/version.rb
 homepage: http://github.com/alphasights/crawl
@@ -78,9 +101,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.11