image_scraper 0.1.5 → 0.1.6

data/VERSION

@@ -1 +1 @@
-0.1.5
+0.1.6

data/image_scraper.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{image_scraper}
-  s.version = "0.1.5"
+  s.version = "0.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John McAliley"]
-  s.date = %q{2011-11-30}
+  s.date = %q{2011-12-22}
   s.description = %q{Simple utility to pull image urls from web page}
   s.email = %q{john.mcaliley@gmail.com}
   s.extra_rdoc_files = [

data/lib/image_scraper/client.rb

@@ -22,6 +22,7 @@ module ImageScraper
       urls = []
       return urls if doc.blank?
       doc.xpath("//img").each do |img|
+        next if img["src"].blank?
         image = URI.escape(img["src"])
         image = ImageScraper::Util.absolute_url(url,image) if convert_to_absolute_url
         urls << image
@@ -41,7 +42,7 @@ module ImageScraper
             image_url
           else
             image_url = ImageScraper::Util.strip_quotes(image_url)
-            @convert_to_absolute_url ? ImageScraper::Util.absolute_url(url,image_url) : image_url
+            @convert_to_absolute_url ? ImageScraper::Util.absolute_url(stylesheet, image_url) : image_url
           end
         end
       end
@@ -51,7 +52,7 @@ module ImageScraper
     def stylesheets
       return [] if doc.blank?
       doc.xpath('//link[@rel="stylesheet"]').collect do |stylesheet|
-        URI.escape ImageScraper::Util.absolute_url(url,stylesheet['href'])
+        ImageScraper::Util.absolute_url url, URI.escape(stylesheet['href'])
       end
     end
   end

data/lib/image_scraper/util.rb

@@ -1,10 +1,18 @@
 module ImageScraper
   module Util
     def self.absolute_url(url,asset=nil)
-      return url if asset.nil?
-      return asset if asset.include?("://")
-      return domain(url)+asset if asset[0]=="/"
-      return domain(url) =~ /\/$/  ? domain(url)+asset : domain(url)+"/"+asset
+      # TODO - what happens when an index redirect occurs?
+      # Example: 'http://example.com/about' specified as url
+      #          'style.css' specified as asset
+      #          url redirects to 'http://example.com/about/'
+      #          and serves http://example.com/about/index.html
+      #          which then links to the relative asset path 'style.css'
+      #          based on original url (http://example.com/about), 
+      #          self.absolute_url gives
+      #          'http://example.com/style.css
+      #          but should get:
+      #          'http://example.com/about/style.css
+      URI.parse(url).merge(URI.parse asset.to_s).to_s
     end
     
     def self.domain(url)
@@ -21,4 +29,4 @@ module ImageScraper
       image_url.gsub("'","").gsub('"','')
     end
   end
-end
+end

data/test/test_image_scraper.rb

@@ -7,9 +7,9 @@ require 'helper'
 
 class TestImageScraper < Test::Unit::TestCase
   should "return list of all image urls on a web page with absolute paths" do
-    images = ["http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/vector/images/search-ltr.png?303-4",
-     "http://en.wikipedia.org//bits.wikimedia.org/images/wikimedia-button.png",
-     "http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/common/images/poweredby_mediawiki_88x31.png"]
+    images = ["http://bits.wikimedia.org/skins-1.18/vector/images/search-ltr.png?303-4",
+     "http://bits.wikimedia.org/images/wikimedia-button.png",
+     "http://bits.wikimedia.org/skins-1.18/common/images/poweredby_mediawiki_88x31.png"]
     scraper = ImageScraper::Client.new("http://en.wikipedia.org/wiki/Standard_test_image",:include_css_images=>false)
     assert_equal images, scraper.image_urls
   end
@@ -72,22 +72,23 @@ class TestImageScraper < Test::Unit::TestCase
   end
 
   should "Handle a URL with unescaped spaces" do
-    images = ["http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/vector/images/search-ltr.png?303-4",
-     "http://en.wikipedia.org//bits.wikimedia.org/images/wikimedia-button.png",
-     "http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/common/images/poweredby_mediawiki_88x31.png"]
+    images = ["http://bits.wikimedia.org/skins-1.18/vector/images/search-ltr.png?303-4",
+     "http://bits.wikimedia.org/images/wikimedia-button.png",
+     "http://bits.wikimedia.org/skins-1.18/common/images/poweredby_mediawiki_88x31.png"]
     scraper = ImageScraper::Client.new("http://en.wikipedia.org/wiki/Standard test image",:include_css_images=>false)
     assert_equal images, scraper.image_urls
   end
   
   should "Handle a page image with an unescaped url" do
     scraper = ImageScraper::Client.new ''
-    scraper.doc = Nokogiri::HTML("<img src='http://test.com/unescaped path'")
+    scraper.doc = Nokogiri::HTML("<img src='http://test.com/unescaped path'>")
     assert_equal ['http://test.com/unescaped%20path'], scraper.page_images
   end 
   
   should "Handle a stylesheet with an unescaped url" do
     scraper = ImageScraper::Client.new ''
-    scraper.doc = Nokogiri::HTML("<link rel='stylesheet' href='http://test.com/unescaped path.css'")
+    scraper.url = 'http://test.com'
+    scraper.doc = Nokogiri::HTML("<link rel='stylesheet' href='http://test.com/unescaped path.css'>")
     assert_equal ['http://test.com/unescaped%20path.css'], scraper.stylesheets
   end 
   
@@ -95,4 +96,9 @@ class TestImageScraper < Test::Unit::TestCase
     scraper = ImageScraper::Client.new 'https://raw.github.com/charlotte-ruby/image_scraper/master/test/resources/stylesheet_unescaped_image.html', :include_css_images => true
     assert_equal ['https://raw.github.com/charlotte-ruby/image_scraper/master/some%20image.png'], scraper.stylesheet_images
   end 
-end
+  
+  should "Handle a stylesheet image with a relative url" do
+    scraper = ImageScraper::Client.new 'https://raw.github.com/charlotte-ruby/image_scraper/master/test/resources/relative_image_url.html', :include_css_images => true
+    assert_equal ['https://raw.github.com/charlotte-ruby/image_scraper/master/test/images/some_image.png'], scraper.stylesheet_images
+  end 
+end

metadata

@@ -5,9 +5,9 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 5
+  - 6
   segments_generated: true
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors: 
 - John McAliley
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-30 00:00:00 -05:00
+date: 2011-12-22 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -185,7 +185,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -4020311873679732909
+      hash: -3072759905091488701
       segments: 
       - 0
       segments_generated: true