cobweb 0.0.52 → 0.0.53

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.52
+h1. Cobweb v0.0.53
 
 h2. Intro 
   

data/lib/cobweb_version.rb

@@ -1,6 +1,6 @@
 class CobwebVersion
   def self.version
-    "0.0.52"
+    "0.0.53"
   end
 
 end

data/lib/content_link_parser.rb

@@ -42,6 +42,7 @@ class ContentLinkParser
     data = link_data
     links = data.keys.map{|key| data[key]}.flatten.uniq
     links = links.map{|link| UriHelper.join_no_fragment(@url, link).to_s }
+    links = links.reject{|link| link =~ /([^\/]+?)\/([^\/]+?)\/.*?\1\/\2/ }    
     links = links.select{|link| options[:valid_schemes].include? link.split(':')[0].to_sym}
     links
   end

data/spec/cobweb/content_link_parser_spec.rb

@@ -29,7 +29,7 @@ describe ContentLinkParser do
         end
         it "should return the correct links" do
           links = @content_parser.links
-          links.length.should == 7
+          links.length.should == 11
         end
       end
       describe "returning image links" do
@@ -92,12 +92,12 @@ describe ContentLinkParser do
         link_data.should be_an_instance_of Hash
 
         link_data.keys.length.should == 5
-        link_data[:links].length.should == 7
+        link_data[:links].length.should == 11
       end
       
       it "should return all http and https links by default" do
         links = @content_parser.all_links
-        links.count.should == 11
+        links.count.should == 13
       end
       
       it "should return all http and https links by default" do
@@ -110,6 +110,13 @@ describe ContentLinkParser do
         links = @content_parser.all_links(:valid_schemes => [:https])
         links.count.should == 1
       end
+      
+      it "should detect and not return link loops" do
+        links = @content_parser.all_links
+        links.should include("http://www.ge.com/repeated1/repeated2/nothing/repeated1/")
+        links.should_not include("http://www.ge.com/repeated1/repeated2/nothing/repeated1/repeated2")
+        links.should include("http://www.ge.com/repeated1/repeated2/nothing/repeated1/asdf/repeated2")
+      end
     end
 
     describe "ignoring default tags" do

data/spec/samples/sample_html_links.html

@@ -27,6 +27,11 @@
   <a href="mailto:stewart@theizone.co.uk">Click Here to email </a>
   <a href="javascript:alert('javascript clicked');">click here for javscript</a>
   <a href="https://sampleurl-a.com/">Click Here for SSL link to URL 1</a>
+  <a href="http://www.ge.com/repeated1/repeated2/nothing/repeated1/"></a>
+  <a href="http://www.ge.com/repeated1/repeated2/nothing/repeated1/repeated2"></a>
+  <a href="http://www.ge.com/repeated1/repeated2/nothing/repeated1/asdf/repeated2"></a>
+  
+  <a href="http://www.ge.com/citizenship/our-priorities/our-products-services/news/citizenship-in-the-news/about-citizenship/our-impact/about-citizenship/performance-against-commitments/metrics/environment-health-safety/about-citizenship/our-impact/metrics/suppliers/about-citizenship/leadership-messages/about-citizenship/awards-recognition/our-commitment-areas/our-people/our-commitment-areas/our-customers/metrics/suppliers/about-citizenship/leadership-messages/our-commitment-areas/our-suppliers/metrics/ecomagination-data/about-citizenship/awards-recognition/news/videos/news/news/videos/about-citizenship/awards-recognition/">looped link</a>
 <frameset><frame src="http://sampleurl-frame.com/"></frame></frameset>
 
 <map id="testmap"><area href="http://sampleurl-area"></area>></map>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.52
+  version: 0.0.53
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70175437227100 !ruby/object:Gem::Requirement
+  requirement: &70165685677520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437227100
+  version_requirements: *70165685677520
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70175437225800 !ruby/object:Gem::Requirement
+  requirement: &70165685676500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437225800
+  version_requirements: *70165685676500
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70175437224640 !ruby/object:Gem::Requirement
+  requirement: &70165685675420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437224640
+  version_requirements: *70165685675420
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70175437223160 !ruby/object:Gem::Requirement
+  requirement: &70165685673680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437223160
+  version_requirements: *70165685673680
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70175437220680 !ruby/object:Gem::Requirement
+  requirement: &70165685671300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437220680
+  version_requirements: *70165685671300
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70175437243360 !ruby/object:Gem::Requirement
+  requirement: &70165685694460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437243360
+  version_requirements: *70165685694460
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70175437241140 !ruby/object:Gem::Requirement
+  requirement: &70165685692120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437241140
+  version_requirements: *70165685692120
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70175437252920 !ruby/object:Gem::Requirement
+  requirement: &70165685703520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437252920
+  version_requirements: *70165685703520
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70175437251920 !ruby/object:Gem::Requirement
+  requirement: &70165685702520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70175437251920
+  version_requirements: *70165685702520
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70175437250760 !ruby/object:Gem::Requirement
+  requirement: &70165685701400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70175437250760
+  version_requirements: *70165685701400
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com