validate-website 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c732e2a061a486000368013967fdd598da0b29a
-  data.tar.gz: 9977f374877aa34f18a3740faf79e8f77f8ed474
+  metadata.gz: 7c6d263126db73e28c33fddb7ba58358f522d305
+  data.tar.gz: bc4d5405e1872d425dea1691456159d523b9c2d2
 SHA512:
-  metadata.gz: 70fab85d94c458bf16a36a4780249ba3d7cfd2d591208a3266f74535520084ce3f9a922c633c46fd47e41ce73b3f113f3f1aef88571d0767e3d452fe33e83b83
-  data.tar.gz: 3416c45ea279abf768d44cefeece27f2e8eb0bcb0d6552f67e0d8e91d914533270a72687fe4136f27b64ca0f72dd215de59dbedc64b851d9daff11ff81475834
+  metadata.gz: 3039cb7c6d5082a2d69915ddf2c1d102362049e5297899eb2aeba0b8b493edc32301aa3803a20411bbfb82bbb9fe05cbebc514083ae1aef9ae954769cd308521
+  data.tar.gz: 14682a11d7e68083aefecd1ab2ca978de5c3d257f0a8d6d5bcacf4427586f5814947f7ea0c3b52bd821d72c3e4dbc0803ae9c950b629576fa10bf05facec1758

data/lib/validate_website/core.rb

@@ -7,7 +7,7 @@ require 'validate_website/option_parser'
 require 'validate_website/validator'
 require 'validate_website/colorful_messages'
 
-require 'anemone'
+require 'spidr'
 
 module ValidateWebsite
 
@@ -15,7 +15,7 @@ module ValidateWebsite
   class Core
 
     attr_accessor :site
-    attr_reader :options, :anemone
+    attr_reader :options, :crawler
 
     include ColorfulMessages
 
@@ -52,29 +52,35 @@ module ValidateWebsite
     #
     def crawl(opts={})
       opts = @options.merge(opts)
+      opts.merge!(:ignore_links => Regexp.new(opts[:exclude])) if opts[:exclude]
+
       puts color(:note, "validating #{@site}", opts[:color]) unless opts[:quiet]
       puts color(:warning, "No internet connection") unless internet_connection?
 
-      @anemone = Anemone.crawl(@site, opts) do |anemone|
-        anemone.skip_links_like Regexp.new(opts[:exclude]) if opts[:exclude]
+      @crawler = Spidr.site(@site, opts) do |crawler|
+        crawler.every_css_page do |page|
+          extract_urls_from_css(page).each do |u|
+            crawler.enqueue(u)
+          end
+        end
 
-        # select the links on each page to follow (iframe, link, css url)
-        anemone.focus_crawl { |page|
-          page.links.concat(extract_urls(page))
-        }
+        crawler.every_html_page do |page|
+          extract_imgs_from_page(page).each do |i|
+            crawler.enqueue(i)
+          end
 
-        anemone.on_every_page { |page|
-          url = page.url.to_s
-          if opts[:markup_validation] && page.html? && page.fetched?
-            validate(page.doc, page.body, url, opts)
+          if opts[:markup_validation] && page.html?
+            validate(page.doc, page.body, page.url, opts)
           end
+        end
 
-          if opts[:not_found] && page.not_found?
+        crawler.every_failed_url do |url|
+          if opts[:not_found]
             @not_found_error = true
-            puts color(:error, "%s linked in %s but not exist" % [url, page.referer], opts[:color])
+            puts color(:error, "%s linked but not exist" % [url], opts[:color])
             to_file(url)
           end
-        }
+        end
       end
     end
 
@@ -93,17 +99,14 @@ module ValidateWebsite
       files.each do |f|
         next unless File.file?(f)
 
-        page = Anemone::Page.new(URI.parse(opts[:site] + URI.encode(f)),
-                                 :body => open(f).read,
-                                 :headers => {'content-type' => ['text/html', 'application/xhtml+xml']})
+        response = fake_http_response(open(f).read)
+        page = Spidr::Page.new(URI.parse(opts[:site] + URI.encode(f)), response)
 
         if opts[:markup_validation]
           validate(page.doc, page.body, f)
         end
         if opts[:not_found]
-          links = page.links
-          links.concat extract_urls_from_img_script_iframe_link(page)
-          check_static_not_found(links.uniq)
+          check_static_not_found(page.links)
         end
       end
     end
@@ -128,13 +131,6 @@ module ValidateWebsite
       end
     end
 
-    def get_url(page, elem, attrname)
-      u = elem.attributes[attrname].to_s
-      return if u.nil? || u.empty?
-      abs = page.to_absolute(u) rescue nil
-      abs if abs && page.in_domain?(abs)
-    end
-
     # check files linked on static document
     # see lib/validate_website/runner.rb
     def check_static_not_found(links, opts={})
@@ -143,8 +139,8 @@ module ValidateWebsite
         file_location = URI.parse(File.join(Dir.getwd, l.path)).path
         # Check CSS url()
         if File.exists?(file_location) && File.extname(file_location) == '.css'
-          css_page = Anemone::Page.new(l, :body => File.read(file_location),
-                                       :headers => {'content-type' => ['text/css']})
+          response = fake_http_response(open(file_location).read, ['text/css'])
+          css_page = Spidr::Page.new(l, response)
           links.concat extract_urls_from_css(css_page)
           links.uniq!
         end
@@ -156,42 +152,29 @@ module ValidateWebsite
       end
     end
 
-    # Extract urls from img script iframe and link element
-    #
-    # @param [Anemone::Page] an Anemone::Page object
-    # @return [Array] Lists of urls
-    #
-    def extract_urls_from_img_script_iframe_link(page)
-      links = Set.new
-      page.doc.css('img, script, iframe, link').each do |elem|
-        if elem.name == 'link'
-          url = get_url(page, elem, "href")
-        else
-          url = get_url(page, elem, "src")
-        end
-        links << url unless url.nil? || url.to_s.empty?
-      end
-      links
-    end
-
     # Extract urls from CSS page
     #
-    # @param [Anemone::Page] an Anemone::Page object
+    # @param [Spidr::Page] an Spidr::Page object
     # @return [Array] Lists of urls
     #
     def extract_urls_from_css(page)
-      page.body.scan(/url\((['".\/\w-]+)\)/).inject([]) do |result, url|
+      page.body.scan(/url\((['".\/\w-]+)\)/).inject(Set[]) do |result, url|
         url = url.first.gsub("'", "").gsub('"', '')
         abs = page.to_absolute(URI.parse(url))
         result << abs
       end
     end
 
-    def extract_urls(page)
-      links = Set.new
-      links.merge extract_urls_from_img_script_iframe_link(page) if page.html?
-      links.merge extract_urls_from_css(page) if page.content_type == 'text/css'
-      links.to_a
+    # Extract imgs urls from page
+    #
+    # @param [Spidr::Page] an Spidr::Page object
+    # @return [Array] Lists of urls
+    #
+    def extract_imgs_from_page(page)
+      page.doc.search('//img[@src]').inject(Set[]) do |result, elem|
+        u = elem.attributes['src']
+        result << page.to_absolute(URI.parse(u))
+      end
     end
 
     ##
@@ -220,5 +203,20 @@ module ValidateWebsite
       end
     end
 
+    # Fake http response for Spidr static crawling
+    # see https://github.com/ruby/ruby/blob/trunk/lib/net/http/response.rb
+    #
+    # @param [String] response body
+    # @param [Array] content types
+    # @return [Net::HTTPResponse] fake http response
+    def fake_http_response(body, content_types=['text/html', 'text/xhtml+xml'])
+      response = Net::HTTPResponse.new '1.1', 200, 'OK'
+      response.instance_variable_set(:@read, true)
+      response.body = body
+      content_types.each do |c|
+        response.add_field('content-type', c)
+      end
+      response
+    end
   end
 end

data/lib/validate_website/option_parser.rb

@@ -16,11 +16,6 @@ module ValidateWebsite
       :color             => true,
       # internal verbose for ValidateWebsite
       :validate_verbose  => false,
-      # Anemone options see anemone/lib/anemone/core.rb
-      :verbose           => false,
-      :cookies           => nil,
-      :accept_cookies    => true,
-      :redirect_limit    => 0,
     }
 
     DEFAULT_OPTS_CRAWL = {
@@ -101,10 +96,6 @@ module ValidateWebsite
              "Only report errors (Default: #{@@default_opts[:quiet]})") { |v|
           options[:quiet] = v
         }
-        o.on("-d", "--debug",
-             "Show anemone log (Default: #{@@default_opts[:verbose]})") { |v|
-          options[:verbose] = v
-        }
 
         o.separator ""
         o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }

data/man/man1/validate-website-static.1

@@ -1,13 +1,13 @@
 '\" t
 .\"     Title: validate-website-static
 .\"    Author: [see the "AUTHOR" section]
-.\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/>
-.\"      Date: 04/05/2012
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\"      Date: 09/20/2014
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "VALIDATE\-WEBSITE\-S" "1" "04/05/2012" "\ \&" "\ \&"
+.TH "VALIDATE\-WEBSITE\-S" "1" "09/20/2014" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------

data/man/man1/validate-website.1

@@ -1,13 +1,13 @@
 '\" t
 .\"     Title: validate-website
 .\"    Author: [see the "AUTHOR" section]
-.\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/>
-.\"      Date: 04/05/2012
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\"      Date: 09/20/2014
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "VALIDATE\-WEBSITE" "1" "04/05/2012" "\ \&" "\ \&"
+.TH "VALIDATE\-WEBSITE" "1" "09/20/2014" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -45,7 +45,7 @@ http://localhost:3000/)
 .PP
 \fB\-u\fR, \fB\-\-user\-agent\fR \fIUSERAGENT\fR
 .RS 4
-Change user agent (Default: Anemone/VERSION)
+Change user agent (Default: Spidr\&.user_agent)
 .RE
 .PP
 \fB\-e\fR, \fB\-\-exclude\fR \fIEXCLUDE\fR
@@ -95,11 +95,6 @@ Show detail of validator errors (Default: false)\&.
 Only report errors (Default: false)\&.
 .RE
 .PP
-\fB\-d\fR, \fB\-\-debug\fR
-.RS 4
-Show anemone log (Default: false)
-.RE
-.PP
 \fB\-h\fR, \fB\-\-help\fR
 .RS 4
 Show help message and exit\&.

data/spec/core_spec.rb

@@ -5,7 +5,8 @@ describe ValidateWebsite::Core do
 
   before do
     WebMock.reset!
-    stub_request(:get, ValidateWebsite::Core::PING_URL).with(:status => 200)
+    stub_request(:get, ValidateWebsite::Core::PING_URL).to_return(:status => 200)
+    stub_request(:get, /#{SPEC_DOMAIN}/).to_return(:status => 200)
     @validate_website = ValidateWebsite::Core.new(:color => false)
   end
 
@@ -18,7 +19,7 @@ describe ValidateWebsite::Core do
                           :content_type => 'text/html')
       @validate_website.site = page.url
       @validate_website.crawl(:quiet => true)
-      @validate_website.anemone.pages.size.must_equal 5
+      @validate_website.crawler.history.size.must_equal 5
     end
 
     it 'extract link' do
@@ -29,7 +30,7 @@ describe ValidateWebsite::Core do
                           :content_type => 'text/html')
       @validate_website.site = page.url
       @validate_website.crawl(:quiet => true)
-      @validate_website.anemone.pages.size.must_equal 98
+      @validate_website.crawler.history.size.must_equal 98
     end
   end
 
@@ -43,7 +44,7 @@ describe ValidateWebsite::Core do
                                     :content_type => 'text/css')
       @validate_website.site = page.url
       @validate_website.crawl(:quiet => true)
-      @validate_website.anemone.pages.size.must_equal 5
+      @validate_website.crawler.history.size.must_equal 5
     end
 
     it "should extract url with single quote" do
@@ -52,7 +53,7 @@ describe ValidateWebsite::Core do
                           :content_type => 'text/css')
       @validate_website.site = page.url
       @validate_website.crawl(:quiet => true)
-      @validate_website.anemone.pages.size.must_equal 2
+      @validate_website.crawler.history.size.must_equal 2
     end
 
     it "should extract url with double quote" do
@@ -61,7 +62,7 @@ describe ValidateWebsite::Core do
                           :content_type => 'text/css')
       @validate_website.site = page.url
       @validate_website.crawl(:quiet => true)
-      @validate_website.anemone.pages.size.must_equal 2
+      @validate_website.crawler.history.size.must_equal 2
     end
   end
 

data/spec/spec_helper.rb

@@ -1,7 +1,7 @@
 # encoding: UTF-8
 require 'minitest/autorun'
 require_relative 'webmock_helper'
-require 'anemone'
+require 'spidr'
 require 'pry'
 
 require 'validate_website/core'

data/spec/validator_spec.rb

@@ -4,7 +4,7 @@ require File.expand_path('../spec_helper', __FILE__)
 describe ValidateWebsite::Validator do
   before do
     WebMock.reset!
-    @http = Anemone::HTTP.new
+    @http = Spidr::Agent.new
   end
 
   describe("xhtml1") do
@@ -15,7 +15,7 @@ describe ValidateWebsite::Validator do
       page = FakePage.new(name,
                           :body => open(file).read,
                           :content_type => 'text/html')
-      @xhtml1_page = @http.fetch_page(page.url)
+      @xhtml1_page = @http.get_page(page.url)
       validator = ValidateWebsite::Validator.new(@xhtml1_page.doc, @xhtml1_page.body)
       validator.dtd.system_id.must_equal dtd_uri
       validator.namespace.must_equal name
@@ -36,7 +36,7 @@ describe ValidateWebsite::Validator do
         page = FakePage.new(name,
                             :body => open(file).read,
                             :content_type => 'text/html')
-        @html5_page = @http.fetch_page(page.url)
+        @html5_page = @http.get_page(page.url)
         validator = ValidateWebsite::Validator.new(@html5_page.doc, @html5_page.body)
         validator.valid?.must_equal true
       end
@@ -46,7 +46,7 @@ describe ValidateWebsite::Validator do
         page = FakePage.new(name,
                             :body => open(file).read,
                             :content_type => 'text/html')
-        @html5_page = @http.fetch_page(page.url)
+        @html5_page = @http.get_page(page.url)
         validator = ValidateWebsite::Validator.new(@html5_page.doc, @html5_page.body)
         validator.valid?.must_equal true
       end
@@ -61,7 +61,7 @@ describe ValidateWebsite::Validator do
         page = FakePage.new(name,
                             :body => open(file).read,
                             :content_type => 'text/html')
-        @html5_page = @http.fetch_page(page.url)
+        @html5_page = @http.get_page(page.url)
       end
 
       it 'should have an array of errors' do
@@ -85,7 +85,7 @@ describe ValidateWebsite::Validator do
       page = FakePage.new(name,
                           :body => open(file).read,
                           :content_type => 'text/html')
-      @html4_strict_page = @http.fetch_page(page.url)
+      @html4_strict_page = @http.get_page(page.url)
       validator = ValidateWebsite::Validator.new(@html4_strict_page.doc, @html4_strict_page.body)
       validator.valid?.must_equal true
     end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: validate-website
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Laurent Arnoud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-18 00:00:00.000000000 Z
+date: 2014-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: anemone
+  name: spidr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: paint
   requirement: !ruby/object:Gem::Requirement
@@ -250,7 +250,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- anemone
+- spidr
 - rainbow
 - multipart_body
 rubyforge_project: