image_scraper 0.1.4 → 0.1.5

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.1.5

data/image_scraper.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{image_scraper}
-  s.version = "0.1.4"
+  s.version = "0.1.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John McAliley"]
-  s.date = %q{2011-07-03}
+  s.date = %q{2011-11-30}
   s.description = %q{Simple utility to pull image urls from web page}
   s.email = %q{john.mcaliley@gmail.com}
   s.extra_rdoc_files = [

data/lib/image_scraper/client.rb

@@ -4,12 +4,12 @@ module ImageScraper
     
     def initialize(url,options={})
       options.reverse_merge!(:convert_to_absolute_url=>true,:include_css_images=>true, :include_css_data_images=>false)
-      @url = url
+      @url = URI.escape(url)
       @convert_to_absolute_url = options[:convert_to_absolute_url]
       @include_css_images = options[:include_css_images]
       @include_css_data_images = options[:include_css_data_images]
-      html = open(url).read
-      @doc = Nokogiri::HTML(html)
+      html = open(@url).read rescue nil
+      @doc = html ? Nokogiri::HTML(html) : nil
     end
   
     def image_urls
@@ -20,8 +20,9 @@ module ImageScraper
     
     def page_images
       urls = []
+      return urls if doc.blank?
       doc.xpath("//img").each do |img|
-        image = img["src"]
+        image = URI.escape(img["src"])
         image = ImageScraper::Util.absolute_url(url,image) if convert_to_absolute_url
         urls << image
       end
@@ -35,7 +36,7 @@ module ImageScraper
         css = file.string rescue IO.read(file)
         
         images += css.scan(/url\((.*?)\)/).collect do |image_url|
-          image_url = image_url[0]
+          image_url = URI.escape image_url[0]
           if image_url.include?("data:image") and @include_css_data_images
             image_url
           else
@@ -48,9 +49,10 @@ module ImageScraper
     end
     
     def stylesheets
+      return [] if doc.blank?
       doc.xpath('//link[@rel="stylesheet"]').collect do |stylesheet|
-        ImageScraper::Util.absolute_url(url,stylesheet['href'])
+        URI.escape ImageScraper::Util.absolute_url(url,stylesheet['href'])
       end
     end
   end
-end
+end

data/test/test_image_scraper.rb

@@ -3,23 +3,21 @@ require 'helper'
 
 
 #TODO: these tests will not work forever.  Try to test against a static web page instead of external URLs
+# Consider using https://raw.github.com/charlotte-ruby/image_scraper urls
+
 class TestImageScraper < Test::Unit::TestCase
   should "return list of all image urls on a web page with absolute paths" do
-    images = ["http://upload.wikimedia.org/wikipedia/en/thumb/2/24/Lenna.png/200px-Lenna.png",
-     "http://bits.wikimedia.org/skins-1.17/common/images/magnify-clip.png",
-     "http://bits.wikimedia.org/skins-1.17/vector/images/search-ltr.png?301-3",
-     "http://bits.wikimedia.org/images/wikimedia-button.png",
-     "http://bits.wikimedia.org/skins-1.17/common/images/poweredby_mediawiki_88x31.png"]
+    images = ["http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/vector/images/search-ltr.png?303-4",
+     "http://en.wikipedia.org//bits.wikimedia.org/images/wikimedia-button.png",
+     "http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/common/images/poweredby_mediawiki_88x31.png"]
     scraper = ImageScraper::Client.new("http://en.wikipedia.org/wiki/Standard_test_image",:include_css_images=>false)
     assert_equal images, scraper.image_urls
   end
 
   should "return list of all image urls on a web page with relative paths" do
-    images = ["http://upload.wikimedia.org/wikipedia/en/thumb/2/24/Lenna.png/200px-Lenna.png",
-     "http://bits.wikimedia.org/skins-1.17/common/images/magnify-clip.png",
-     "http://bits.wikimedia.org/skins-1.17/vector/images/search-ltr.png?301-3",
-     "http://bits.wikimedia.org/images/wikimedia-button.png",
-     "http://bits.wikimedia.org/skins-1.17/common/images/poweredby_mediawiki_88x31.png"]
+    images = ["//bits.wikimedia.org/skins-1.18/vector/images/search-ltr.png?303-4",
+     "//bits.wikimedia.org/images/wikimedia-button.png",
+     "//bits.wikimedia.org/skins-1.18/common/images/poweredby_mediawiki_88x31.png"]
     scraper = ImageScraper::Client.new("http://en.wikipedia.org/wiki/Standard_test_image",:convert_to_absolute_url=>false,:include_css_images=>false)
     assert_equal images, scraper.image_urls
   end
@@ -27,7 +25,7 @@ class TestImageScraper < Test::Unit::TestCase
   should "return list of stylesheets contained in html page (relative path)" do
     doc = Nokogiri::HTML(IO.read(File.dirname(__FILE__)+"/resources/stylesheet_test.html"))
     domain = "http://test.com"
-    assert_equal ["http://test.com/phoenix/testcentral.css"], ImageScraper::Client.new("http://test.com").stylesheets
+    assert_equal ["http://test.com/phoenix/testcentral.css","http://test.com/engine1/style.css"], ImageScraper::Client.new("http://test.com").stylesheets
   end
   
   should "return proper absolute url for a page and asset" do
@@ -43,8 +41,8 @@ class TestImageScraper < Test::Unit::TestCase
   end
   
   should "return images from a stylesheet" do
-    scraper = ImageScraper::Client.new("http://local.couponshack.com")
-    assert scraper.stylesheet_images.include? ("http://local.couponshack.com/images/bg.png")
+    scraper = ImageScraper::Client.new("http://couponshack.com")
+    assert scraper.stylesheet_images.include? ("http://couponshack.com/images/bg.jpg")
   end
   
   should "strip quotes from a url" do
@@ -59,4 +57,42 @@ class TestImageScraper < Test::Unit::TestCase
     assert_equal "http://ug.ly", ImageScraper::Util.domain("http://ug.ly/what")
     assert_equal "http://www.ug.ly", ImageScraper::Util.domain("http://www.ug.ly/what/is/this/")
   end
+  
+  should "return nil for doc if URL is invalid" do
+    scraper = ImageScraper::Client.new("couponshack.com")
+    assert scraper.doc.nil?
+  end
+  
+  should "return empty arrays if URL is invalid" do
+    scraper = ImageScraper::Client.new("couponshack.com")
+    assert_equal [], scraper.image_urls
+    assert_equal [], scraper.stylesheets
+    assert_equal [], scraper.stylesheet_images
+    assert_equal [], scraper.page_images
+  end
+
+  should "Handle a URL with unescaped spaces" do
+    images = ["http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/vector/images/search-ltr.png?303-4",
+     "http://en.wikipedia.org//bits.wikimedia.org/images/wikimedia-button.png",
+     "http://en.wikipedia.org//bits.wikimedia.org/skins-1.18/common/images/poweredby_mediawiki_88x31.png"]
+    scraper = ImageScraper::Client.new("http://en.wikipedia.org/wiki/Standard test image",:include_css_images=>false)
+    assert_equal images, scraper.image_urls
+  end
+  
+  should "Handle a page image with an unescaped url" do
+    scraper = ImageScraper::Client.new ''
+    scraper.doc = Nokogiri::HTML("<img src='http://test.com/unescaped path'")
+    assert_equal ['http://test.com/unescaped%20path'], scraper.page_images
+  end 
+  
+  should "Handle a stylesheet with an unescaped url" do
+    scraper = ImageScraper::Client.new ''
+    scraper.doc = Nokogiri::HTML("<link rel='stylesheet' href='http://test.com/unescaped path.css'")
+    assert_equal ['http://test.com/unescaped%20path.css'], scraper.stylesheets
+  end 
+  
+  should "Handle a stylesheet image with an unescaped url" do
+    scraper = ImageScraper::Client.new 'https://raw.github.com/charlotte-ruby/image_scraper/master/test/resources/stylesheet_unescaped_image.html', :include_css_images => true
+    assert_equal ['https://raw.github.com/charlotte-ruby/image_scraper/master/some%20image.png'], scraper.stylesheet_images
+  end 
 end

metadata

@@ -5,8 +5,9 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 4
-  version: 0.1.4
+  - 5
+  segments_generated: true
+  version: 0.1.5
 platform: ruby
 authors: 
 - John McAliley
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-03 00:00:00 -04:00
+date: 2011-11-30 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,6 +27,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
+        segments_generated: true
         version: "0"
   type: :runtime
   prerelease: false
@@ -39,6 +41,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
+        segments_generated: true
         version: "0"
   type: :runtime
   prerelease: false
@@ -52,6 +55,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
+        segments_generated: true
         version: "0"
   type: :runtime
   prerelease: false
@@ -65,6 +69,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
+        segments_generated: true
         version: "0"
   type: :development
   prerelease: false
@@ -80,6 +85,7 @@ dependencies:
         - 1
         - 0
         - 0
+        segments_generated: true
         version: 1.0.0
   type: :development
   prerelease: false
@@ -95,6 +101,7 @@ dependencies:
         - 1
         - 5
         - 2
+        segments_generated: true
         version: 1.5.2
   type: :development
   prerelease: false
@@ -108,6 +115,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
+        segments_generated: true
         version: "0"
   type: :development
   prerelease: false
@@ -121,6 +129,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
+        segments_generated: true
         version: "0"
   type: :runtime
   prerelease: false
@@ -134,6 +143,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
+        segments_generated: true
         version: "0"
   type: :runtime
   prerelease: false
@@ -175,9 +185,10 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -830537873008219940
+      hash: -4020311873679732909
       segments: 
       - 0
+      segments_generated: true
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
@@ -186,6 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version 
       segments: 
       - 0
+      segments_generated: true
       version: "0"
 requirements: []