image_scraper 0.1.0 → 0.1.1

data/Gemfile

@@ -2,9 +2,11 @@ source "http://rubygems.org"
 # Add dependencies required to use your gem here.
 # Example:
 #   gem "activesupport", ">= 2.3.5"
+
 gem "nokogiri"
 gem "css_parser"
 gem "rails"
+
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do

data/Rakefile

@@ -20,7 +20,8 @@ Jeweler::Tasks.new do |gem|
   gem.email = "john.mcaliley@gmail.com"
   gem.authors = ["John McAliley"]
   gem.add_dependency "nokogiri"
-  gem.add_dependency "css_parser"  
+  gem.add_dependency "css_parser"
+  gem.files.exclude "test/**/*"
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/image_scraper.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{image_scraper}
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John McAliley"]
@@ -19,16 +19,15 @@ Gem::Specification.new do |s|
   s.files = [
     ".document",
     "Gemfile",
-    "Gemfile.lock",
     "LICENSE.txt",
     "README.md",
     "Rakefile",
     "VERSION",
     "image_scraper.gemspec",
     "lib/image_scraper.rb",
-    "test/helper.rb",
-    "test/resources/stylesheet_test.html",
-    "test/test_image_scraper.rb"
+    "lib/image_scraper/client.rb",
+    "lib/image_scraper/railtie.rb",
+    "lib/image_scraper/util.rb"
   ]
   s.homepage = %q{http://github.com/charlotte-ruby/image_scraper}
   s.licenses = ["MIT"]

data/lib/image_scraper.rb

@@ -3,78 +3,6 @@ require 'rails'
 require 'open-uri'
 require 'nokogiri'
 
-module ImageScraper
-  class Client
-    attr_accessor :url, :convert_to_absolute_url, :include_css_images, :include_css_data_images, :doc
-    
-    def initialize(url,options={})
-      options.reverse_merge!(:convert_to_absolute_url=>true,:include_css_images=>true, :include_css_data_images=>false)
-      @url = url
-      @convert_to_absolute_url = options[:convert_to_absolute_url]
-      @include_css_images = options[:include_css_images]
-      @include_css_data_images = options[:include_css_data_images]
-      @doc = Nokogiri::HTML(open url)
-    end
-  
-    def image_urls
-      images = page_images
-      images += stylesheet_images if include_css_images
-      images
-    end
-    
-    def page_images
-      urls = []
-      doc.xpath("//img").each do |img|
-        image = img["src"]
-        image = ImageScraper::Util.absolute_url(url,image) if convert_to_absolute_url
-        urls << image
-      end
-      urls
-    end
-    
-    def stylesheet_images
-      images = []
-      stylesheets.each do |stylesheet|
-        file = open(stylesheet)
-        css = file.string rescue IO.read(file)
-        
-        images += css.scan(/url\((.*?)\)/).collect do |image_url|
-          if image_url.include?("data:image") and @include_css_data_images
-            image_url[0]
-          else
-            @convert_to_absolute_url ? ImageScraper::Util.absolute_url(url,image_url[0]) : image_url
-          end
-        end
-      end
-      images
-    end
-    
-    def stylesheets
-      doc.xpath('//link[@rel="stylesheet"]').collect do |stylesheet|
-        ImageScraper::Util.absolute_url(url,stylesheet['href'])
-      end
-    end
-  end
-  
-  module Util
-    def self.absolute_url(url,asset=nil)
-      return domain(url) + path(url) if asset.nil? and asset.include("://")
-      return asset if asset.include?("://")
-      return domain(url)+asset if asset[0]=="/"
-      return domain(url) =~ /\/$/  ? domain(url)+asset : domain(url)+"/"+asset
-    end
-    
-    def self.domain(url)
-      uri = URI.parse(url)
-      "#{uri.scheme}://#{uri.host}"
-    end
-    
-    def self.path(url)
-      uri = URI.parse(url)
-      uri.path
-    end
-  end
-  
-  class Railtie < Rails::Railtie
-  end
-end
+require 'image_scraper/railtie'
+require 'image_scraper/util'
+require 'image_scraper/client'

data/lib/image_scraper/client.rb

@@ -0,0 +1,53 @@
+module ImageScraper
+  class Client
+    attr_accessor :url, :convert_to_absolute_url, :include_css_images, :include_css_data_images, :doc
+    
+    def initialize(url,options={})
+      options.reverse_merge!(:convert_to_absolute_url=>true,:include_css_images=>true, :include_css_data_images=>false)
+      @url = url
+      @convert_to_absolute_url = options[:convert_to_absolute_url]
+      @include_css_images = options[:include_css_images]
+      @include_css_data_images = options[:include_css_data_images]
+      @doc = Nokogiri::HTML(open url)
+    end
+  
+    def image_urls
+      images = page_images
+      images += stylesheet_images if include_css_images
+      images
+    end
+    
+    def page_images
+      urls = []
+      doc.xpath("//img").each do |img|
+        image = img["src"]
+        image = ImageScraper::Util.absolute_url(url,image) if convert_to_absolute_url
+        urls << image
+      end
+      urls
+    end
+    
+    def stylesheet_images
+      images = []
+      stylesheets.each do |stylesheet|
+        file = open(stylesheet)
+        css = file.string rescue IO.read(file)
+        
+        images += css.scan(/url\((.*?)\)/).collect do |image_url|
+          if image_url.include?("data:image") and @include_css_data_images
+            image_url[0]
+          else
+            @convert_to_absolute_url ? ImageScraper::Util.absolute_url(url,image_url[0]) : image_url
+          end
+        end
+      end
+      images
+    end
+    
+    def stylesheets
+      doc.xpath('//link[@rel="stylesheet"]').collect do |stylesheet|
+        ImageScraper::Util.absolute_url(url,stylesheet['href'])
+      end
+    end
+  end
+end

data/lib/image_scraper/railtie.rb

@@ -0,0 +1,7 @@
+require 'rails'
+require 'image_scraper'
+
+module ImageScraper
+  class Railtie < Rails::Railtie
+  end
+end

data/lib/image_scraper/util.rb

@@ -0,0 +1,20 @@
+module ImageScraper
+  module Util
+    def self.absolute_url(url,asset=nil)
+      return domain(url) + path(url) if asset.nil? and asset.include("://")
+      return asset if asset.include?("://")
+      return domain(url)+asset if asset[0]=="/"
+      return domain(url) =~ /\/$/  ? domain(url)+asset : domain(url)+"/"+asset
+    end
+    
+    def self.domain(url)
+      uri = URI.parse(url)
+      "#{uri.scheme}://#{uri.host}"
+    end
+    
+    def self.path(url)
+      uri = URI.parse(url)
+      uri.path
+    end
+  end
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 0
-  version: 0.1.0
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - John McAliley
@@ -150,15 +150,16 @@ extra_rdoc_files:
 files: 
 - .document
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - VERSION
 - image_scraper.gemspec
 - lib/image_scraper.rb
+- lib/image_scraper/client.rb
+- lib/image_scraper/railtie.rb
+- lib/image_scraper/util.rb
 - test/helper.rb
-- test/resources/stylesheet_test.html
 - test/test_image_scraper.rb
 has_rdoc: true
 homepage: http://github.com/charlotte-ruby/image_scraper
@@ -174,7 +175,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 44385371017236818
+      hash: 1275395306475000737
       segments: 
       - 0
       version: "0"

data/Gemfile.lock

@@ -1,86 +0,0 @@
-GEM
-  remote: http://rubygems.org/
-  specs:
-    abstract (1.0.0)
-    actionmailer (3.0.7)
-      actionpack (= 3.0.7)
-      mail (~> 2.2.15)
-    actionpack (3.0.7)
-      activemodel (= 3.0.7)
-      activesupport (= 3.0.7)
-      builder (~> 2.1.2)
-      erubis (~> 2.6.6)
-      i18n (~> 0.5.0)
-      rack (~> 1.2.1)
-      rack-mount (~> 0.6.14)
-      rack-test (~> 0.5.7)
-      tzinfo (~> 0.3.23)
-    activemodel (3.0.7)
-      activesupport (= 3.0.7)
-      builder (~> 2.1.2)
-      i18n (~> 0.5.0)
-    activerecord (3.0.7)
-      activemodel (= 3.0.7)
-      activesupport (= 3.0.7)
-      arel (~> 2.0.2)
-      tzinfo (~> 0.3.23)
-    activeresource (3.0.7)
-      activemodel (= 3.0.7)
-      activesupport (= 3.0.7)
-    activesupport (3.0.7)
-    arel (2.0.10)
-    builder (2.1.2)
-    css_parser (1.1.9)
-    erubis (2.6.6)
-      abstract (>= 1.0.0)
-    git (1.2.5)
-    i18n (0.5.0)
-    jeweler (1.5.2)
-      bundler (~> 1.0.0)
-      git (>= 1.2.5)
-      rake
-    mail (2.2.19)
-      activesupport (>= 2.3.6)
-      i18n (>= 0.4.0)
-      mime-types (~> 1.16)
-      treetop (~> 1.4.8)
-    mime-types (1.16)
-    nokogiri (1.4.4)
-    polyglot (0.3.1)
-    rack (1.2.3)
-    rack-mount (0.6.14)
-      rack (>= 1.0.0)
-    rack-test (0.5.7)
-      rack (>= 1.0)
-    rails (3.0.7)
-      actionmailer (= 3.0.7)
-      actionpack (= 3.0.7)
-      activerecord (= 3.0.7)
-      activeresource (= 3.0.7)
-      activesupport (= 3.0.7)
-      bundler (~> 1.0)
-      railties (= 3.0.7)
-    railties (3.0.7)
-      actionpack (= 3.0.7)
-      activesupport (= 3.0.7)
-      rake (>= 0.8.7)
-      thor (~> 0.14.4)
-    rake (0.9.0)
-    rcov (0.9.9)
-    shoulda (2.11.3)
-    thor (0.14.6)
-    treetop (1.4.9)
-      polyglot (>= 0.3.1)
-    tzinfo (0.3.27)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.0.0)
-  css_parser
-  jeweler (~> 1.5.2)
-  nokogiri
-  rails
-  rcov
-  shoulda

data/test/resources/stylesheet_test.html

@@ -1,17 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
-   "http://www.w3.org/TR/html4/strict.dtd">
-
-<html lang="en">
-<head>
-	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-	<title>stylesheet_test</title>
-	<meta name="generator" content="TextMate http://macromates.com/">
-	<meta name="author" content="John McAliley">
-	<link rel="stylesheet" href="/css/master.css" type="text/css" media="screen" title="no title" charset="utf-8">
-	<link rel="stylesheet" href="/css/master2.css" type="text/css" media="screen" title="no title" charset="utf-8">
-	<!-- Date: 2011-06-01 -->
-</head>
-<body>
-
-</body>
-</html>