rawler 0.0.6 → 0.0.7

data/Gemfile.lock

@@ -3,7 +3,12 @@ GEM
   specs:
     diff-lcs (1.1.2)
     fakeweb (1.3.0)
+    hoe (2.6.2)
+      rake (>= 0.8.7)
+      rubyforge (>= 2.0.4)
+    json_pure (1.5.1)
     nokogiri (1.4.4)
+    rake (0.8.7)
     rspec (2.4.0)
       rspec-core (~> 2.4.0)
       rspec-expectations (~> 2.4.0)
@@ -12,11 +17,14 @@ GEM
     rspec-expectations (2.4.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.4.0)
+    rubyforge (2.0.4)
+      json_pure (>= 1.1.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   fakeweb (= 1.3.0)
+  hoe (= 2.6.2)
   nokogiri (= 1.4.4)
   rspec (= 2.4.0)

data/lib/rawler/base.rb

@@ -7,7 +7,7 @@ module Rawler
     def initialize(url, output, username=nil, password=nil)
       @responses = {}
 
-      Rawler.url      = url
+      Rawler.url      = URI.escape(url)
       Rawler.output   = Logger.new(output)
       Rawler.username = username
       Rawler.password = password

data/lib/rawler/crawler.rb

@@ -16,7 +16,7 @@ module Rawler
       response = Rawler::Request.get(url)
       
       doc = Nokogiri::HTML(response.body)
-      doc.css('a').map { |a| a['href'] }.map { |url| absolute_url(url) }.select { |url| valid_url?(url) }
+      doc.css('a').map { |a| a['href'] }.select { |url| !url.nil? }.map { |url| absolute_url(url) }.select { |url| valid_url?(url) }
     rescue Errno::ECONNREFUSED
       write("Couldn't connect to #{url}")
       []
@@ -28,9 +28,11 @@ module Rawler
     private
     
     def absolute_url(path)
-      path.strip!
+      path = URI.encode(path.strip)
       if path[0].chr == '/'
         URI.parse(url).merge(path.to_s).to_s
+      elsif URI.parse(path).scheme.nil?
+        URI.parse(url).merge("/#{path.to_s}").to_s
       else
         path
       end

data/lib/rawler.rb

@@ -7,7 +7,7 @@ require 'logger'
 require 'rawler/core_extensions'
 
 module Rawler
-  VERSION = '0.0.6'
+  VERSION = '0.0.7'
   
   mattr_accessor :output
   mattr_accessor :url

data/spec/lib/rawler/crawler_spec.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 require File.dirname(__FILE__) + '/../../spec_helper.rb'
 
 describe Rawler::Crawler do
@@ -36,14 +38,14 @@ describe Rawler::Crawler do
     
     let(:url)     { 'http://example.com/path' }
     let(:crawler) { Rawler::Crawler.new(url) }
-    let(:content) { '<a href="/foo">foo</a>' }
+    let(:content) { '<a href="/foo">foo</a> <a href="bar">bar</a>' }
     
     before(:each) do
       register(url, content)
     end
     
     it "should parse relative links" do
-      crawler.links.should == ['http://example.com/foo']
+      crawler.links.should == ['http://example.com/foo', 'http://example.com/bar']
     end
     
   end
@@ -75,8 +77,24 @@ describe Rawler::Crawler do
       register(url, content)
     end
     
-    it "should parse relative links" do
-      crawler.links.should == ['http://example.com/foo#bar']
+    it "should parse urls with hashtags" do
+      crawler.links.should == ['http://example.com/foo%23bar']
+    end
+    
+  end
+  
+  context "urls with unicode characters" do
+    
+    let(:url)     { 'http://example.com' }
+    let(:crawler) { Rawler::Crawler.new(url) }
+    let(:content) { '<a href="http://example.com/写程序容易出现的几个不好的地方">foo</a>' }
+    
+    before(:each) do
+      register(url, content)
+    end
+
+    it "should parse unicode links" do
+      crawler.links.should == ['http://example.com/%E5%86%99%E7%A8%8B%E5%BA%8F%E5%AE%B9%E6%98%93%E5%87%BA%E7%8E%B0%E7%9A%84%E5%87%A0%E4%B8%AA%E4%B8%8D%E5%A5%BD%E7%9A%84%E5%9C%B0%E6%96%B9']
     end
     
   end
@@ -85,7 +103,7 @@ describe Rawler::Crawler do
     let(:url)     { 'http://example.com/path' }
     let(:crawler) { Rawler::Crawler.new(url) }
     let(:js_url)  { "javascript:fn('nbjmup;jhfs.esf{fio/dpn');" }
-    let(:content) { "<a href=\"#{js_url}\">foo</a>" }
+    let(:content) { "<a href=\"#{js_url}\">foo</a><a name=\"foo\">" }
     
     before(:each) do
       register(url, content)
@@ -94,9 +112,9 @@ describe Rawler::Crawler do
     it "should parse relative links" do
       crawler.links.should == []
     end
-    
+
     it "should report the error" do
-      crawler.should_receive(:write).with("Invalid url - #{js_url}")
+      crawler.should_receive(:write).with("Invalid url - javascript:fn('nbjmup;jhfs.esf%7Bfio/dpn');")
       crawler.links
     end
   end

data/spec/lib/rawler_spec.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 require File.dirname(__FILE__) + '/../spec_helper.rb'
 
 describe Rawler::Base do
@@ -9,6 +11,16 @@ describe Rawler::Base do
     Rawler.stub!(:output).and_return(output)
     register('http://example.com', site)
   end
+
+  describe "url encoding" do
+    it "should encode url" do
+      original = 'http://example.com/写程序容易出现的几个不好的地方'
+      expected = 'http://example.com/%E5%86%99%E7%A8%8B%E5%BA%8F%E5%AE%B9%E6%98%93%E5%87%BA%E7%8E%B0%E7%9A%84%E5%87%A0%E4%B8%AA%E4%B8%8D%E5%A5%BD%E7%9A%84%E5%9C%B0%E6%96%B9'
+
+      Rawler::Base.new(original, output)
+      Rawler.url.should == expected
+    end
+  end
   
   describe "validate_links" do
     
@@ -50,16 +62,7 @@ describe Rawler::Base do
       
       rawler.validate
     end
-    
-    it "should validate links with #hashtags" do
-      register('http://example.com/foo1', '<a href="http://example.com/page-with#hashtag">x</a>')
-      register('http://example.com/page-with', '')
-      
-      output.should_receive(:info).with('200 - http://example.com/page-with#hashtag')
-      
-      rawler.validate
-    end
-            
+                
   end
   
   describe "get_status_code" do
@@ -200,4 +203,4 @@ describe Rawler::Base do
     site
   end
   
-end
+end

data/specs.watchr

@@ -25,7 +25,6 @@ def run(files_to_run)
 end
 
 def run_all_tests
-  puts "foo"
   run(all_test_files.join(' '))
 end
 

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: rawler
 version: !ruby/object:Gem::Version 
-  hash: 19
-  prerelease: 
+  prerelease: false
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 7
+  version: 0.0.7
 platform: ruby
 authors: 
 - Oscar Del Ben
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-04 00:00:00 +01:00
+date: 2011-03-07 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,28 +25,41 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: rubyforge
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 47
         segments: 
         - 2
-        - 8
         - 0
-        version: 2.8.0
+        - 4
+        version: 2.0.4
   type: :development
   version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 6
+        - 2
+        version: 2.6.2
+  type: :development
+  version_requirements: *id003
 description: |-
   Rawler is a Ruby library that crawls your website and checks the status code for each of your links. Useful for finding dead links.
   
@@ -100,7 +112,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -109,14 +120,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: oscardelben
-rubygems_version: 1.4.1
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Rawler is a Ruby library that crawls your website and checks the status code for each of your links