image_scraper 0.0.2 → 0.1.0

data/Gemfile

@@ -3,6 +3,8 @@ source "http://rubygems.org"
 # Example:
 #   gem "activesupport", ">= 2.3.5"
 gem "nokogiri"
+gem "css_parser"
+gem "rails"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do

data/Gemfile.lock

@@ -1,22 +1,86 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    abstract (1.0.0)
+    actionmailer (3.0.7)
+      actionpack (= 3.0.7)
+      mail (~> 2.2.15)
+    actionpack (3.0.7)
+      activemodel (= 3.0.7)
+      activesupport (= 3.0.7)
+      builder (~> 2.1.2)
+      erubis (~> 2.6.6)
+      i18n (~> 0.5.0)
+      rack (~> 1.2.1)
+      rack-mount (~> 0.6.14)
+      rack-test (~> 0.5.7)
+      tzinfo (~> 0.3.23)
+    activemodel (3.0.7)
+      activesupport (= 3.0.7)
+      builder (~> 2.1.2)
+      i18n (~> 0.5.0)
+    activerecord (3.0.7)
+      activemodel (= 3.0.7)
+      activesupport (= 3.0.7)
+      arel (~> 2.0.2)
+      tzinfo (~> 0.3.23)
+    activeresource (3.0.7)
+      activemodel (= 3.0.7)
+      activesupport (= 3.0.7)
+    activesupport (3.0.7)
+    arel (2.0.10)
+    builder (2.1.2)
+    css_parser (1.1.9)
+    erubis (2.6.6)
+      abstract (>= 1.0.0)
     git (1.2.5)
+    i18n (0.5.0)
     jeweler (1.5.2)
       bundler (~> 1.0.0)
       git (>= 1.2.5)
       rake
+    mail (2.2.19)
+      activesupport (>= 2.3.6)
+      i18n (>= 0.4.0)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.16)
     nokogiri (1.4.4)
+    polyglot (0.3.1)
+    rack (1.2.3)
+    rack-mount (0.6.14)
+      rack (>= 1.0.0)
+    rack-test (0.5.7)
+      rack (>= 1.0)
+    rails (3.0.7)
+      actionmailer (= 3.0.7)
+      actionpack (= 3.0.7)
+      activerecord (= 3.0.7)
+      activeresource (= 3.0.7)
+      activesupport (= 3.0.7)
+      bundler (~> 1.0)
+      railties (= 3.0.7)
+    railties (3.0.7)
+      actionpack (= 3.0.7)
+      activesupport (= 3.0.7)
+      rake (>= 0.8.7)
+      thor (~> 0.14.4)
     rake (0.9.0)
     rcov (0.9.9)
     shoulda (2.11.3)
+    thor (0.14.6)
+    treetop (1.4.9)
+      polyglot (>= 0.3.1)
+    tzinfo (0.3.27)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.0.0)
+  css_parser
   jeweler (~> 1.5.2)
   nokogiri
+  rails
   rcov
   shoulda

data/README.md

@@ -14,17 +14,23 @@ Install w/ Bundler
     
 ## USAGE
 
-Pull all the image URLs from a specified URL and convert all images URLs to absolute paths
+Initialize the image scraper client
 
-    ImageScraper.image_urls("http://www.rubygems.org")
+    image_scraper = ImageScraper::Client.new("http://www.rubygems.org")
 
-Pull all images URLs from specified URL and use relative URLs
+You can also pass an options hash to the client when you initialize it:
 
-    ImageScraper.image_urls("http://www.rubygems.org",false)
+    image_scraper = ImageScraper::Client.new("http://www.rubygems.org", options)
+    # OPTIONS - If you don't pass the option, it will default to true
+    # :convert_to_absolute_url - If there are relative image URLS, it will convert them to absolute URLS.
+    # :include_css_images - If there are stylesheets on the page, it will pull images out of the stylesheet.  For example: background: url(/images/some-image.png).
+    # :include_css_data_images - Will include data images from CSS.  For example: data:image/gif;base64,R0lGODlhEAAOALMAAOazToeH............
+    
+Get the images from the url specified when you initialized the client:
 
-## TODO
+    image_scraper.image_urls
 
-1. Parse CSS files for images
+This will return an array of strings.
 
 ## Contributing to image_scraper
  

data/Rakefile

@@ -20,6 +20,7 @@ Jeweler::Tasks.new do |gem|
   gem.email = "john.mcaliley@gmail.com"
   gem.authors = ["John McAliley"]
   gem.add_dependency "nokogiri"
+  gem.add_dependency "css_parser"  
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.1.0

data/image_scraper.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{image_scraper}
-  s.version = "0.0.2"
+  s.version = "0.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John McAliley"]
-  s.date = %q{2011-05-27}
+  s.date = %q{2011-06-02}
   s.description = %q{Simple utility to pull image urls from web page}
   s.email = %q{john.mcaliley@gmail.com}
   s.extra_rdoc_files = [
@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
     "image_scraper.gemspec",
     "lib/image_scraper.rb",
     "test/helper.rb",
+    "test/resources/stylesheet_test.html",
     "test/test_image_scraper.rb"
   ]
   s.homepage = %q{http://github.com/charlotte-ruby/image_scraper}
@@ -45,26 +46,35 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_runtime_dependency(%q<css_parser>, [">= 0"])
+      s.add_runtime_dependency(%q<rails>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_runtime_dependency(%q<css_parser>, [">= 0"])
     else
       s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<css_parser>, [">= 0"])
+      s.add_dependency(%q<rails>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<css_parser>, [">= 0"])
     end
   else
     s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<css_parser>, [">= 0"])
+    s.add_dependency(%q<rails>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
     s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<css_parser>, [">= 0"])
   end
 end
 

data/lib/image_scraper.rb

@@ -1,20 +1,80 @@
+require 'pp'
+require 'rails'
 require 'open-uri'
 require 'nokogiri'
 
 module ImageScraper
-  class Railtie < Rails::Railtie
+  class Client
+    attr_accessor :url, :convert_to_absolute_url, :include_css_images, :include_css_data_images, :doc
+    
+    def initialize(url,options={})
+      options.reverse_merge!(:convert_to_absolute_url=>true,:include_css_images=>true, :include_css_data_images=>false)
+      @url = url
+      @convert_to_absolute_url = options[:convert_to_absolute_url]
+      @include_css_images = options[:include_css_images]
+      @include_css_data_images = options[:include_css_data_images]
+      @doc = Nokogiri::HTML(open url)
+    end
+  
+    def image_urls
+      images = page_images
+      images += stylesheet_images if include_css_images
+      images
+    end
+    
+    def page_images
+      urls = []
+      doc.xpath("//img").each do |img|
+        image = img["src"]
+        image = ImageScraper::Util.absolute_url(url,image) if convert_to_absolute_url
+        urls << image
+      end
+      urls
+    end
+    
+    def stylesheet_images
+      images = []
+      stylesheets.each do |stylesheet|
+        file = open(stylesheet)
+        css = file.string rescue IO.read(file)
+        
+        images += css.scan(/url\((.*?)\)/).collect do |image_url|
+          if image_url.include?("data:image") and @include_css_data_images
+            image_url[0]
+          else
+            @convert_to_absolute_url ? ImageScraper::Util.absolute_url(url,image_url[0]) : image_url
+          end
+        end
+      end
+      images
+    end
+    
+    def stylesheets
+      doc.xpath('//link[@rel="stylesheet"]').collect do |stylesheet|
+        ImageScraper::Util.absolute_url(url,stylesheet['href'])
+      end
+    end
   end
-
-  def self.image_urls(url, convert_to_absolute_url=true)
-    uri = URI.parse(url)
-    domain = "#{uri.scheme}://#{uri.host}"
-    doc = Nokogiri::HTML(open url)
-    urls = []
-    doc.xpath("//img").each do |img|
-      image = img["src"]
-      image = domain + image if convert_to_absolute_url and !image.include?("://")
-      urls << image
-    end
-    urls
+  
+  module Util
+    def self.absolute_url(url,asset=nil)
+      return domain(url) + path(url) if asset.nil? and asset.include("://")
+      return asset if asset.include?("://")
+      return domain(url)+asset if asset[0]=="/"
+      return domain(url) =~ /\/$/  ? domain(url)+asset : domain(url)+"/"+asset
+    end
+    
+    def self.domain(url)
+      uri = URI.parse(url)
+      "#{uri.scheme}://#{uri.host}"
+    end
+    
+    def self.path(url)
+      uri = URI.parse(url)
+      uri.path
+    end
+  end
+  
+  class Railtie < Rails::Railtie
   end
 end

data/test/resources/stylesheet_test.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+   "http://www.w3.org/TR/html4/strict.dtd">
+
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+	<title>stylesheet_test</title>
+	<meta name="generator" content="TextMate http://macromates.com/">
+	<meta name="author" content="John McAliley">
+	<link rel="stylesheet" href="/css/master.css" type="text/css" media="screen" title="no title" charset="utf-8">
+	<link rel="stylesheet" href="/css/master2.css" type="text/css" media="screen" title="no title" charset="utf-8">
+	<!-- Date: 2011-06-01 -->
+</head>
+<body>
+
+</body>
+</html>

data/test/test_image_scraper.rb

@@ -1,13 +1,47 @@
+require 'pp'
 require 'helper'
 
+
+#TODO: these tests will not work forever.  Try to test against a static web page instead of external URLs
 class TestImageScraper < Test::Unit::TestCase
   should "return list of all image urls on a web page with absolute paths" do
-    images = ["http://upload.wikimedia.org/wikipedia/en/thumb/2/24/Lenna.png/200px-Lenna.png", "http://bits.wikimedia.org/skins-1.17/common/images/magnify-clip.png", "http://bits.wikimedia.org/skins-1.17/vector/images/search-ltr.png?301-2", "http://en.wikipedia.org/images/wikimedia-button.png", "http://bits.wikimedia.org/skins-1.17/common/images/poweredby_mediawiki_88x31.png"]
-    assert_equal ImageScraper.image_urls("http://en.wikipedia.org/wiki/Standard_test_image"), images
+    images = ["http://upload.wikimedia.org/wikipedia/en/thumb/2/24/Lenna.png/200px-Lenna.png",
+     "http://bits.wikimedia.org/skins-1.17/common/images/magnify-clip.png",
+     "http://bits.wikimedia.org/skins-1.17/vector/images/search-ltr.png?301-2",
+     "http://en.wikipedia.org/images/wikimedia-button.png",
+     "http://bits.wikimedia.org/skins-1.17/common/images/poweredby_mediawiki_88x31.png"]
+    scraper = ImageScraper::Client.new("http://en.wikipedia.org/wiki/Standard_test_image",:include_css_images=>false)
+    assert_equal images, scraper.image_urls
   end
-  
+
   should "return list of all image urls on a web page with relative paths" do
-    images = ["http://upload.wikimedia.org/wikipedia/en/thumb/2/24/Lenna.png/200px-Lenna.png","http://bits.wikimedia.org/skins-1.17/common/images/magnify-clip.png","http://bits.wikimedia.org/skins-1.17/vector/images/search-ltr.png?301-2","/images/wikimedia-button.png","http://bits.wikimedia.org/skins-1.17/common/images/poweredby_mediawiki_88x31.png"]
-    assert_equal ImageScraper.image_urls("http://en.wikipedia.org/wiki/Standard_test_image",false), images
-  end  
-end
+    images = ["http://upload.wikimedia.org/wikipedia/en/thumb/2/24/Lenna.png/200px-Lenna.png",
+     "http://bits.wikimedia.org/skins-1.17/common/images/magnify-clip.png",
+     "http://bits.wikimedia.org/skins-1.17/vector/images/search-ltr.png?301-2",
+     "/images/wikimedia-button.png",
+     "http://bits.wikimedia.org/skins-1.17/common/images/poweredby_mediawiki_88x31.png"]
+    scraper = ImageScraper::Client.new("http://en.wikipedia.org/wiki/Standard_test_image",:convert_to_absolute_url=>false,:include_css_images=>false)
+    assert_equal images, scraper.image_urls
+  end
+
+  should "return list of stylesheets contained in html page (relative path)" do
+    doc = Nokogiri::HTML(IO.read(File.dirname(__FILE__)+"/resources/stylesheet_test.html"))
+    domain = "http://test.com"
+    assert_equal ["http://test.com/phoenix/testcentral.css"], ImageScraper::Client.new("http://test.com").stylesheets
+  end
+  
+  should "return proper absolute url for a page and asset" do
+    assert_equal "http://www.test.com/image.gif", ImageScraper::Util.absolute_url("http://www.test.com","image.gif")
+    assert_equal "http://www.test.com/images/image.gif",ImageScraper::Util.absolute_url("http://www.test.com","images/image.gif")
+    assert_equal "http://www.test.com/images/image.gif",ImageScraper::Util.absolute_url("http://www.test.com","/images/image.gif")
+    assert_equal "http://www.test.com/image.gif", ImageScraper::Util.absolute_url("http://www.test.com/","image.gif")
+    assert_equal "http://www.test.com/images/image.gif", ImageScraper::Util.absolute_url("http://www.test.com/","/images/image.gif")
+    assert_equal "http://www.test.com/images/image.gif", ImageScraper::Util.absolute_url("http://www.test.com/","images/image.gif")
+    assert_equal "http://www.test.com/images/image.gif", ImageScraper::Util.absolute_url("http://www.test.com/","/images/image.gif")
+  end
+  
+  should "return images from a stylesheet" do
+    scraper = ImageScraper::Client.new("http://local.couponshack.com")
+    assert scraper.stylesheet_images.include? ("http://local.couponshack.com/images/bg.png")
+  end
+end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 2
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors: 
 - John McAliley
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-27 00:00:00 -04:00
+date: 2011-06-02 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -31,7 +31,7 @@ dependencies:
   prerelease: false
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: shoulda
+  name: css_parser
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -40,12 +40,38 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: bundler
+  name: rails
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -57,10 +83,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *id003
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -72,10 +98,10 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *id004
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: rcov
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -85,10 +111,10 @@ dependencies:
         version: "0"
   type: :development
   prerelease: false
-  version_requirements: *id005
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency 
   name: nokogiri
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -98,7 +124,20 @@ dependencies:
         version: "0"
   type: :runtime
   prerelease: false
-  version_requirements: *id006
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: css_parser
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id009
 description: Simple utility to pull image urls from web page
 email: john.mcaliley@gmail.com
 executables: []
@@ -119,6 +158,7 @@ files:
 - image_scraper.gemspec
 - lib/image_scraper.rb
 - test/helper.rb
+- test/resources/stylesheet_test.html
 - test/test_image_scraper.rb
 has_rdoc: true
 homepage: http://github.com/charlotte-ruby/image_scraper
@@ -134,7 +174,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -3054709322002535321
+      hash: 44385371017236818
       segments: 
       - 0
       version: "0"