crawl 0.0.4 → 0.0.5

data/README.md

@@ -0,0 +1,15 @@
+# Crawl
+
+Crawl pages witin a domain, reporting any page that returns a bad response code
+
+Usage:
+
+    >crawl [options] domain
+
+      -s, --start /home,/about         Starting path(s), defaults to /
+      -u, --username username          Basic auth username
+      -p, --password password          Basic auth password
+      -c, --ci                         Output files for CI integration
+      -v, --verbose                    Give details when crawling
+      -m, --markup                     Validate HTML markup
+      -h, --help                       Show this message

data/bin/crawl

@@ -10,6 +10,7 @@ optparse = OptionParser.new do |opts|
   opts.on('-p', '--password password', String, 'Basic auth password') { |o| options[:password] = o }
   opts.on('-c', '--ci', 'Output files for CI integration') { |o| options[:ci] = o }
   opts.on('-v', '--verbose', 'Give details when crawling') { |o| options[:verbose] = o }
+  opts.on('-m', '--markup', 'Validate markup') { |o| options[:markup] = o }
   opts.on_tail("-h", "--help", "Show this message") { |o| puts opts; exit }
 end.parse!
 

data/lib/crawl/engine.rb

@@ -33,6 +33,7 @@ class Crawl::Engine
     @verbose = options[:verbose] || ENV['VERBOSE']
     @number_of_dots = 0
     @report_manager = CI::Reporter::ReportManager.new("crawler") if options[:ci]
+    @validate_markup = options[:markup]
   end
 
   def run
@@ -43,7 +44,7 @@ class Crawl::Engine
         next unless response.headers[:content_type] =~ %r{text/html}
         @visited_documents << link
         @found_links += links = find_links(link, response.to_str)
-        # validate(link, response.body_str)
+        validate(link, response.body) if @validate_markup
       end
     end
   end
@@ -87,7 +88,6 @@ private
     error_messages = messages.select { |message| message['type'] != 'info' }
 
     if error_messages.empty?
-      handle_success
       true
     else
       response = error_messages.map do |message|
@@ -97,7 +97,6 @@ private
       end.join("\n\n")
 
       @errors << Result.new(link, response)
-      handle_error('I')
       false
     end
   rescue RestClient::ServiceUnavailable
@@ -132,7 +131,7 @@ private
     test_suite.finish
     @report_manager.write_report(test_suite) if options[:ci]
     return response
-  rescue RestClient::InternalServerError, RestClient::ResourceNotFound => e
+  rescue RestClient::InternalServerError, RestClient::ResourceNotFound, RestClient::Unauthorized => e
     @errors << Result.new(link, "Error whilst retrieving page: #{e.message}")
     @invalid_links << link
     return nil

data/lib/crawl/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Crawl
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawl
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-10 00:00:00.000000000 Z
+date: 2012-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70243809291280 !ruby/object:Gem::Requirement
+  requirement: &70216317741600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70243809291280
+  version_requirements: *70216317741600
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &70243809290740 !ruby/object:Gem::Requirement
+  requirement: &70216317740600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70243809290740
+  version_requirements: *70216317740600
 - !ruby/object:Gem::Dependency
   name: ci_reporter
-  requirement: &70243809290140 !ruby/object:Gem::Requirement
+  requirement: &70216317739980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70243809290140
+  version_requirements: *70216317739980
 description: Crawl all pages on a domain, checking for errors
 email:
 - tor@alphasights.com
@@ -54,7 +54,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
-- README
+- README.md
 - Rakefile
 - bin/crawl
 - crawl.gemspec

data/README

@@ -1,8 +0,0 @@
-Crawl pages witin a domain, reporting any page that returns a bad response code
-Usage: crawl [options] domain
-    -s, --start /home,/about         Starting path(s), defaults to /
-    -u, --username username          Basic auth username
-    -p, --password password          Basic auth password
-    -c, --ci                         Output files for CI integration
-    -v, --verbose                    Give details when crawling
-    -h, --help                       Show this message