rawler 0.1.4 → 0.1.5

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.1.5

data/bin/rawler

@@ -18,8 +18,10 @@ EOS
   opt :password, "HTT Basic Password", :type => :string
   opt :wait, "Seconds to wait between requests, may be fractional e.g. '1.5'", :type => :float, :default => 3.0
   opt :log, "Log results to file rawler_log.txt", :type => :boolean, :default => false
+  opt :css, "Check CSS links", :type => :boolean, :default => false
 end
 
+
 domain = ARGV.shift
 
 if domain.nil?

data/lib/rawler.rb

@@ -11,6 +11,7 @@ module Rawler
   mattr_accessor :wait
   mattr_accessor :username, :password
   mattr_accessor :log
+  mattr_accessor :css
 
   autoload :Base, "rawler/base"
   autoload :Crawler, "rawler/crawler"

data/lib/rawler/base.rb

@@ -13,6 +13,7 @@ module Rawler
       Rawler.password = options[:password]
       Rawler.wait     = options[:wait]
       Rawler.log      = options[:log]
+      Rawler.css      = options[:css]
       @logfile = File.new("rawler_log.txt", "w") if Rawler.log
     end
 
@@ -30,10 +31,24 @@ module Rawler
       end
     end
 
+    def validate_css_links_in_page(page)
+      Rawler::Crawler.new(page).css_links.each do |page_url|
+        validate_non_html(page_url, page)
+        sleep(Rawler.wait)
+      end
+    end
+
     def validate_page(page_url, from_url)
       if not_yet_parsed?(page_url)
         add_status_code(page_url, from_url) 
         validate_links_in_page(page_url) if same_domain?(page_url)
+        validate_css_links_in_page(page_url) if same_domain?(page_url) and Rawler.css
+      end
+    end
+
+    def validate_non_html(page_url, from_url)
+      if not_yet_parsed?(page_url)  
+        add_status_code(page_url, from_url)
       end
     end
 

data/lib/rawler/crawler.rb

@@ -27,6 +27,24 @@ module Rawler
       []
     end
 
+    def css_links
+      if different_domain?(url, Rawler.url) || not_html?(url)
+        return []
+      end
+
+      response = Rawler::Request.get(url)
+
+      doc = Nokogiri::HTML(response.body)
+
+      doc.css('link').map { |a| a['href'] }.select { |url| !url.nil? }.map { |url| absolute_url(url) }.select { |url| valid_url?(url) }
+    rescue Errno::ECONNREFUSED
+      write("Couldn't connect to #{url}")
+      []
+    rescue Errno::ETIMEDOUT
+      write("Connection to #{url} timed out")
+      []
+    end
+
     private
 
     def absolute_url(path)
@@ -50,8 +68,16 @@ module Rawler
       URI.parse(url_1).host != URI.parse(url_2).host
     end
 
+    def content_type(url)
+      Rawler::Request.head(url).content_type
+    end
+
     def not_html?(url)
-      Rawler::Request.head(url).content_type != 'text/html'
+      content_type(url) != 'text/html'
+    end
+
+    def not_css?(url)
+       content_type(url) != 'text/css'
     end
 
     def valid_url?(url)

data/rawler.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "rawler"
-  s.version = "0.1.4"
+  s.version = "0.1.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Oscar Del Ben"]
-  s.date = "2012-02-27"
+  s.date = "2012-04-14"
   s.description = "Rawler is a tool that crawls the links of your website"
   s.email = "info@oscardelben.com"
   s.executables = ["rawler"]
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/oscardelben/rawler"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
+  s.rubygems_version = "1.8.21"
   s.summary = "Rawler is a tool that crawls the links of your website"
 
   if s.respond_to? :specification_version then

data/spec/lib/rawler/crawler_spec.rb

@@ -18,7 +18,8 @@ describe Rawler::Crawler do
     let(:crawler) { Rawler::Crawler.new(url) }
     let(:content) {
       content = <<-content
-        <p><a href="http://example.com/foo">foo</a></p>
+        <link rel="stylesheet" href="css/styles.css" />
+	<p><a href="http://example.com/foo">foo</a></p>
 
     		<p><a href="http://external.com/bar">bar</a></p>
     	content
@@ -32,6 +33,9 @@ describe Rawler::Crawler do
       crawler.links.should == ['http://example.com/foo', 'http://external.com/bar']
     end
     
+    it "should parse css links" do
+      crawler.css_links.should == ['http://example.com/css/styles.css']
+    end 
   end
   
   context "relative paths" do

metadata

@@ -1,134 +1,137 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rawler
-version: !ruby/object:Gem::Version 
-  hash: 19
+version: !ruby/object:Gem::Version
+  version: 0.1.5
   prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 4
-  version: 0.1.4
 platform: ruby
-authors: 
+authors:
 - Oscar Del Ben
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-02-27 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+date: 2012-04-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id001
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
   prerelease: false
-  name: nokogiri
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id002
-  type: :development
-  prerelease: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: fakeweb
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id003
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: rspec
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id004
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: shoulda
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id005 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
-  requirement: *id005
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: bundler
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 1
-        - 6
-        - 4
-        version: 1.6.4
-  requirement: *id006
+      - !ruby/object:Gem::Version
+        version: 1.0.0
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
   name: jeweler
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id007 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id007
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+- !ruby/object:Gem::Dependency
   name: rcov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Rawler is a tool that crawls the links of your website
 email: info@oscardelben.com
-executables: 
+executables:
 - rawler
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.txt
 - README.md
-files: 
+files:
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -153,37 +156,31 @@ files:
 - test/test_rawler.rb
 - vendor/lib-trollop.rb
 homepage: http://github.com/oscardelben/rawler
-licenses: 
+licenses:
 - MIT
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: 2522129833142198431
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Rawler is a tool that crawls the links of your website
 test_files: []
-