crawl 0.1.0.beta2 → 0.1.0.beta3

data/bin/crawl

@@ -8,10 +8,9 @@ optparse = OptionParser.new do |opts|
   opts.on('-s', '--start /home,/about', Array, 'Starting path(s), defaults to /') { |o| options[:start] = o }
   opts.on('-u', '--username username', String, 'Basic auth username') { |o| options[:username] = o }
   opts.on('-p', '--password password', String, 'Basic auth password') { |o| options[:password] = o }
-  opts.on('-c', '--ci', 'Output files for CI integration') { |o| options[:ci] = o }
-  opts.on('-v', '--verbose', 'Give details when crawling') { |o| options[:verbose] = o }
-  opts.on('-m', '--markup', 'Validate markup') { |o| options[:markup] = o }
+  opts.on('-v', '--verbose', 'Give details when crawling') { |o| $verbose = o }
   opts.on_tail("-h", "--help", "Show this message") { |o| puts opts; exit }
+  opts.on_tail("-v", "--version", "Print version") { |o| puts Crawl::VERSION; exit }
 end.parse!
 
 options.merge!(domain: optparse.first)
@@ -33,6 +32,5 @@ crawler.run
 crawler.summarize
 
 if crawler.errors?
-  puts 'Errors during crawling'
   exit -1
 end

data/crawl.gemspec

@@ -16,7 +16,6 @@ Gem::Specification.new do |gem|
   gem.version       = Crawl::VERSION
   gem.add_dependency('nokogiri')
   gem.add_dependency('rest-client')
-  gem.add_dependency('ci_reporter')
-  gem.add_dependency('eventmachine', '~> 1.0.0.beta.4')
+  gem.add_dependency('eventmachine', '~> 1.0.0.rc1')
   gem.add_dependency('em-http-request')
 end

data/lib/crawl/engine.rb

@@ -19,11 +19,11 @@ class Crawl::Engine
   def initialize(caller_options = {})
     @options = DEFAULT_OPTIONS.merge(caller_options)
     @authorization = Base64.encode64("#{options[:username]}:#{options[:password]}")
-    @verbose = options[:verbose] || ENV['VERBOSE']
-    @validate_markup = options[:markup]
-    @register = Crawl::Register.new(options[:start].to_a)
+    @register = Crawl::Register.new
 
-    @report_manager = CI::Reporter::ReportManager.new("crawler") if options[:ci]
+    start_pages = options[:start].to_a.map{|page| Page.new(@register, page, 'the command line')}
+
+    @register.add(start_pages)
   end
 
   def run
@@ -36,10 +36,8 @@ class Crawl::Engine
     return if @register.processing_size >= EM.threadpool_size
     if @register.finished?
       EventMachine.stop
-    elsif (link = @register.next_link)
-      puts "\nChecking #{link}" if @verbose
-      retrieve(link)
-      # validate(link, response.body) if @validate_markup
+    elsif (page = @register.next_page)
+      retrieve(page)
       process_next
     end
   end
@@ -54,100 +52,45 @@ class Crawl::Engine
 
 private
 
-  def validate(link, body)
-    puts "  Validating..." if @verbose
-
-    json_response = RestClient.post 'http://validator.nu?out=json', body, :content_type => 'text/html; charset=utf-8'
-    messages = JSON.parse(json_response.body)['messages']
-    error_messages = messages.select { |message| message['type'] != 'info' }
-
-    if error_messages.empty?
-      true
-    else
-      response = error_messages.map do |message|
-        type, message = message['type'], message['message']
-        type_color = type == 'error' ? 31 : 33
-        "\e[#{type_color};1m" + type.capitalize + "\e[0m: " + message
-      end.join("\n\n")
-
-      @register.error link, response
-      false
-    end
-  rescue RestClient::ServiceUnavailable
-    handle_error('U')
-    false
-  end
-
-  def register_error(link, message)
-    @register.error link, message
-    @register.returned_invalid link
-    process_next
-  end
-
-  def retrieve(link)
-    # test_suite = CI::Reporter::TestSuite.new(link)
-    # test_case  = CI::Reporter::TestCase.new(link)
-    # test_suite.start
-    # test_case.start
-    # test_suite.name = link
-    # test_case.name = link
-
-    puts "Fetching #{options[:domain] + link} ..." if @verbose
+  def retrieve(page)
+    puts "Fetching #{page.url} ..." if $verbose
 
-    unless link.start_with? '/'
-      register_error(link, "Relative path found. Crawl does not support relative paths.")
-      return nil
-    end
+    full_url = options[:domain] + page.url
 
-    http = EventMachine::HttpRequest.new(options[:domain] + link)
+    http = EventMachine::HttpRequest.new(full_url)
     req = http.get :redirects => MAX_REDIRECTS, :head => {'authorization' => [options[:username], options[:password]]}
-    req.timeout(30)
+    req.timeout(15)
 
     req.errback do
       if req.nil?
-         @register.retry(link, 'WAT?')
-         process_next
+         page.intermittent("Req is nil. WAT?")
        elsif msg = req.error
-         register_error(link, msg)
+         page.intermittent(msg)
        elsif req.response.nil? || req.response.empty?
-         @register.retry(link, 'Timeout?')
-         process_next
-         # register_error(link, 'Timeout?')
+         page.intermittent('Timeout?')
        else
-         @register.retry(link, 'Partial response: Server Broke Connection?')
-         process_next
+         page.intermittent('Partial response: Server Broke Connection?')
        end
+       process_next
     end
 
     req.callback do
-      if VALID_RESPONSE_CODES.include?(req.response_header.status)
-        @register.returned link
+      status_code = req.response_header.status
+      if VALID_RESPONSE_CODES.include?(status_code)
+        page.success
         if req.response_header["CONTENT_TYPE"] =~ %r{text/html}
-          @register.add find_links(link, req.response.to_str)
+          @register.add find_linked_pages(page, req.response.to_str)
         end
+      elsif(status_code == 503)
+        page.intermittent("Status code: 503")
       else
-        @register.error link, "Status code was #{req.response_header.status}"
-        @register.returned_broken link
-        # test_case.failures << Crawl::Failure.new(link, req.response_header.status, linked_from(link))
-        # test_suite.testcases << test_case
-        # test_suite.finish
-        # @report_manager.write_report(test_suite) if options[:ci]
+        page.fatal("Status code: #{status_code}")
       end
       process_next
     end
-
-    # test_case.finish
-    # test_suite.testcases << test_case
-    # test_suite.finish
-    # @report_manager.write_report(test_suite) if options[:ci]
   end
 
-  def linked_from(target)
-    @register.source_for target
-  end
-
-  def find_links(source_link, body)
-    puts "  Finding links.." if @verbose
+  def find_linked_pages(page, body)
     doc = Nokogiri::HTML(body)
     anchors = doc.css('a').to_a
     anchors.reject!{|anchor| anchor['onclick'].to_s =~ /f.method = 'POST'/}
@@ -160,11 +103,6 @@ private
     raw_links.map!{|link| link.sub(options[:domain], '')}
     raw_links.delete_if{|link| link =~ %r{^http(s)?://}}
     raw_links.delete_if{|link| IGNORE.any?{|pattern| link =~ pattern}}
-    raw_links.each do |target_link|
-      puts "    Adding #{target_link} found on #{source_link}" if @verbose
-      @register.set_link_source(target_link, source_link)
-    end
-
-    raw_links
+    raw_links.map{ |url| Page.new(@register, url, page.url) }
   end
 end

data/lib/crawl/page.rb

@@ -0,0 +1,53 @@
+class Page
+  include Comparable
+
+  attr_reader :register, :url, :source, :error
+
+  ATTEMPTS = 3
+
+  def initialize(register, url, source)
+    @register = register
+    @url = url
+    @source = source
+    @attempts = 0
+    @errors = nil
+  end
+
+  def <=>(other)
+    url <=> other.url
+  end
+
+  def eql?(other)
+    url.eql?(other.url)
+  end
+
+  def hash
+    url.hash
+  end
+
+  def success
+    @error = nil
+    @register.completed(self)
+  end
+
+  def fatal(error)
+    puts "  Fatal - #{error}" if $VERBOSE
+    @error = error
+    @register.completed(self)
+  end
+
+  def intermittent(error)
+    puts "  Intermittent - #{error}" if $VERBOSE
+    if @attempts >= ATTEMPTS
+      @error = error
+      @register.completed(self)
+    else
+      @attempts += 1
+      @register.retry(self)
+    end
+  end
+
+  def to_s
+    "#{url} found on #{source} - #{error || 'OK'}"
+  end
+end

data/lib/crawl/register.rb

@@ -2,57 +2,38 @@ class Crawl::Register
 
   Result = Struct.new(:url, :object)
 
-  def initialize(unprocessed)
-    @unprocessed = unprocessed
-    @processing = []
-    @processed = []
-
-    @invalid_links = Set[]
-    @broken_pages = Set[]
-
-    @errors = []
-    @link_sources = {}
+  def initialize
+    @unprocessed = Set.new
+    @processing = Set.new
+    @processed = Set.new
   end
 
-  def add(links)
-    new_links = links - @processed - @processing - @unprocessed
-    @unprocessed += new_links
+  def add(pages)
+    new_pages = pages.to_set - @processed - @processing - @unprocessed
+    new_pages.each do |new_page|
+      puts "  Adding #{new_page.url}" if $verbose
+    end
+    @unprocessed.merge(new_pages)
   end
 
-  def next_link
-    link = @unprocessed.shift
-    @processing << link if link
+  def next_page
+    page = @unprocessed.first
+    @unprocessed.delete(page)
+    @processing << page if page
     if @processing.size > EM.threadpool_size
       puts "WARNING: #{@processing.size} pages are being process when EM threadpool only has #{EM.threadpool_size} threads."
     end
-    link
-  end
-
-  def set_link_source(link, source)
-    @link_sources[link] = source
-  end
-
-  def source_for(link)
-    @link_sources.fetch link, '?'
-  end
-
-  def error(link, object)
-    @errors << Result.new(link, object)
+    page
   end
 
-  def returned_invalid(link)
-    returned link
-    @invalid_links << link
+  def retry(page)
+    @unprocessed << page
+    @processing.delete(page)
   end
 
-  def returned_broken(link)
-    returned link
-    @broken_pages << link
-  end
-
-  def returned(link)
-    @processed << link
-    @processing -= [link]
+  def completed(page)
+    @processed << page
+    @processing.delete(page)
   end
 
   def finished?
@@ -63,40 +44,22 @@ class Crawl::Register
     @processing.size
   end
 
-  def retry(link, reason)
-    puts "Retrying #{link} : #{reason}"
-    @processing -= [link]
-    @unprocessed << link
+  def error_pages
+    @processed.select{ |page| page.error }
   end
 
-  def summarize
-    if @errors.size > 0
+  def errors?
+    !error_pages.empty?
+  end
 
-      @errors.each do |error|
-        puts "\n#{error.url}"
-        puts "  Linked from #{source_for error.url}"
-        puts error.object.to_s.word_wrap.split("\n").map{|line| '  ' + line}
+  def summarize
+    if errors?
+      puts "\nPages with errors:"
+      error_pages.each do |page|
+        puts page.to_s
       end
-
-      print(<<-SUM)
-
-Pages crawled: #{@processed.size}
-Pages with errors: #{@errors.size - @invalid_links.size}
-Broken pages: #{@broken_pages.size}
-Invalid links: #{@invalid_links.size}
-
-I=Invalid P=Parse Error S=Status code bad
-
-SUM
-      exit(@errors.size)
     else
-       puts "\n\n#{@processed.size} pages crawled"
+       puts "\n#{@processed.size} pages crawled without errors."
     end
-
-    puts
-  end
-
-  def errors?
-    @errors.size > 0
   end
 end

data/lib/crawl/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Crawl
-  VERSION = "0.1.0.beta2"
+  VERSION = "0.1.0.beta3"
 end

data/lib/crawl.rb

@@ -16,4 +16,5 @@ require_relative "crawl/version"
 require_relative "crawl/engine"
 require_relative "crawl/string"
 require_relative "crawl/failure"
-require_relative "crawl/register"
+require_relative "crawl/register"
+require_relative "crawl/page"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawl
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta2
+  version: 0.1.0.beta3
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-01 00:00:00.000000000 Z
+date: 2012-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70140204969220 !ruby/object:Gem::Requirement
+  requirement: &70174767878200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70140204969220
+  version_requirements: *70174767878200
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &70140204968500 !ruby/object:Gem::Requirement
+  requirement: &70174767877780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,32 +32,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70140204968500
-- !ruby/object:Gem::Dependency
-  name: ci_reporter
-  requirement: &70140204967840 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *70140204967840
+  version_requirements: *70174767877780
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &70140204967240 !ruby/object:Gem::Requirement
+  requirement: &70174767877280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta.4
+        version: 1.0.0.rc1
   type: :runtime
   prerelease: false
-  version_requirements: *70140204967240
+  version_requirements: *70174767877280
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &70140204966740 !ruby/object:Gem::Requirement
+  requirement: &70174767876860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70140204966740
+  version_requirements: *70174767876860
 description: Crawl all pages on a domain, checking for errors
 email:
 - tor@alphasights.com
@@ -83,6 +72,7 @@ files:
 - lib/crawl.rb
 - lib/crawl/engine.rb
 - lib/crawl/failure.rb
+- lib/crawl/page.rb
 - lib/crawl/register.rb
 - lib/crawl/string.rb
 - lib/crawl/version.rb