thumbnail_scraper 0.0.1

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+*.rvmrc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in thumbnail_scraper.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    thumbnail_scraper (0.0.1)
+      fastimage
+      httpclient
+      nokogiri
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.2.8)
+    crack (0.3.1)
+    diff-lcs (1.1.3)
+    fastimage (1.2.13)
+    httpclient (2.2.5)
+    nokogiri (1.5.3)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.1)
+    webmock (1.7.6)
+      addressable (~> 2.2, > 2.2.5)
+      crack (>= 0.1.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rspec
+  thumbnail_scraper!
+  webmock

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jan Filipowski
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+Thumbnail Scraper
+============
+
+detect, fetch and generate a thumbnail for any url
+
+Basic usage
+-------------
+```ruby
+require 'thumbnail_scraper'
+
+include ThumbnailScraper
+scraper = ThumbnailScraper.new
+image = scraper.image_to_thumbnail_url("http://www.monibuds.com/")
+thumbnail_url = image.url
+```
+
+ThumbnailScraper#image_to_thumbnail_url method returns Image object, which contains its size and url.
+
+Suggested usage
+---------------
+
+We encourage you to use it with delayed_job as jobs queue and dragonfly as image storage tool. Your job could look like following:
+
+```ruby
+require 'thumbnail_scraper'
+
+module Jobs
+  class ScrapThumbnailJob < Struct.new(:page)
+    def perform
+      scraper = ::ThumbnailScraper::ThumbnailScraper.new
+      image = scraper.image_to_thumbnail_for_url(page.url)
+      page.thumbnail_url = image.url.to_s
+      page.save!
+    end 
+  end 
+end
+```

data/Rakefile

@@ -0,0 +1,5 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)

data/lib/thumbnail_scraper.rb

@@ -0,0 +1,44 @@
+require "thumbnail_scraper/version"
+require 'thumbnail_scraper/image'
+require 'thumbnail_scraper/http_receiver'
+
+module ThumbnailScraper
+  class ThumbnailScraper
+    attr_accessor :http_receiver
+
+    def initialize(receiver=HttpReceiver.new)
+      @http_receiver = receiver
+    end
+
+    def create_image(url)
+      Image.new(url)
+    end
+
+    def image_to_thumbnail_for_url(url)
+      webpage = http_receiver.receive_webpage(url)
+      if webpage.has_open_graph_image?
+        image = create_image(webpage.open_graph_image_url)
+      elsif webpage.has_linked_image?
+        image = create_image(webpage.linked_image_url)
+      else
+        image = select_best_possible_image_to_scrap(webpage.attached_images_urls)
+      end
+      image
+    end
+
+    def select_best_possible_image_to_scrap(images_urls)
+      images = images_urls.map{|image_url| create_image(image_url)}
+      valid_images = select_valid_images(images)
+      return nil if valid_images.empty?
+      valid_images.max{|a, b| a.area <=> b.area}
+    end
+
+    def select_valid_images(images)
+      images.select{|image| image_is_valid?(image)}
+    end
+
+    def image_is_valid?(image)
+      image.width >= 50 && image.height >= 50 && image.width.to_f / image.height.to_f <= 3 && image.height.to_f / image.width.to_f <= 3
+    end
+  end
+end

data/lib/thumbnail_scraper/http_receiver.rb

@@ -0,0 +1,17 @@
+require 'thumbnail_scraper/webpage'
+require 'httpclient'
+
+module ThumbnailScraper
+  class HttpReceiver
+    attr_accessor :http_client
+
+    def initialize
+      @http_client = ::HTTPClient.new
+    end
+
+    def receive_webpage(url)
+      content = http_client.get_content(url)
+      Webpage.new(url, content)
+    end
+  end
+end

data/lib/thumbnail_scraper/image.rb

@@ -0,0 +1,28 @@
+require 'thumbnail_scraper/with_smart_url'
+require 'fastimage'
+
+module ThumbnailScraper
+  class Image
+    include WithSmartUrl
+
+    def initialize(url)
+      self.url = url
+    end
+
+    def size
+      @size ||= ::FastImage.size(url.to_s)
+    end
+
+    def width
+      size[0]
+    end
+
+    def height
+      size[1]
+    end
+
+    def area
+      width * height
+    end
+  end
+end

data/lib/thumbnail_scraper/version.rb

@@ -0,0 +1,3 @@
+module ThumbnailScraper
+  VERSION = "0.0.1"
+end

data/lib/thumbnail_scraper/webpage.rb

@@ -0,0 +1,65 @@
+require 'thumbnail_scraper/with_smart_url'
+require 'nokogiri'
+
+module ThumbnailScraper
+  class Webpage
+    attr_accessor :body
+
+    include WithSmartUrl
+
+    def initialize(url, body)
+      self.url = url
+      self.body = body
+    end
+
+    def document
+      Nokogiri::HTML::Document.parse(body)
+    end
+
+    def image_url(image_path)
+      if image_path.start_with?("http://") || image_path.start_with?("https://")
+        image_url = URI(image_path)
+      elsif image_path.start_with?("//")
+        image_url = URI(image_path)
+        image_url.scheme = url.scheme
+      else
+        image_url = URI(url.to_s)
+        if Pathname.new(image_path).absolute?
+          image_url.path = image_path
+        else
+          image_url.path = File.expand_path(File.join(File.dirname(url.path), image_path))
+        end
+      end
+      image_url
+    end
+
+    def open_graph_image_url
+      return @open_graph_image_url if defined?(@open_graph_image_url)
+      elements = document.xpath("//meta[@property='og:image']/@content")
+      return nil if elements.empty?
+      image_path = elements.first.value
+      @open_graph_image_url = image_url(image_path)
+    end
+
+    def has_open_graph_image?
+      !open_graph_image_url.nil?
+    end
+
+    def linked_image_url
+      return @linked_image_url if defined?(@linked_image_url)
+      elements = document.xpath("//link[@rel='img_src']/@href")
+      return nil if elements.empty?
+      image_path = elements.first.value
+      @linked_image_url = image_url(image_path)
+    end
+
+    def has_linked_image?
+      !linked_image_url.nil?
+    end
+
+    def attached_images_urls
+      elements = document.xpath("//img/@src")
+      elements.map{|element| image_url(element.value)}
+    end
+  end
+end

data/lib/thumbnail_scraper/with_smart_url.rb

@@ -0,0 +1,17 @@
+require 'uri'
+
+module ThumbnailScraper
+  module WithSmartUrl
+    def url=(value)
+      if value.is_a?(URI)
+        @url = value
+      else
+        @url = URI.parse(value)
+      end
+    end
+
+    def url
+      @url
+    end
+  end
+end

data/spec/sample_pages/images.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <p><img src="/images/subcatalog/smallest.jpg" /></p>
+    <p>Sample</p>
+    <p><img src="/images/kitty.jpg" /></p>
+    <p>Sample</p>
+    <p><img src="images/biggest.jpg" /></p>
+  </body>
+</html>
+

data/spec/sample_pages/img_src.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <link rel="img_src" href="images/kitty.jpg" />
+  </head>
+  <body>
+    <p>Sample</p>
+  </body>
+</html>
+

data/spec/sample_pages/og_image.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <meta property="og:image" content="images/kitty.jpg" />
+  </head>
+  <body>
+    <p>Sample</p>
+  </body>
+</html>

data/spec/thumbnail_scraper_helper.rb

@@ -0,0 +1,11 @@
+require 'webmock/rspec'
+
+def asset_file_path(name)
+  File.join("spec", "sample_pages", name)
+end
+
+def asset_file(name)
+  path = asset_file_path(name)
+  File.new(path)
+end
+

data/spec/thumbnail_scraper_spec.rb

@@ -0,0 +1,52 @@
+require 'thumbnail_scraper_helper'
+require 'thumbnail_scraper'
+
+describe ThumbnailScraper::ThumbnailScraper do
+  before :each do
+    @thumbnail_scraper = ThumbnailScraper::ThumbnailScraper.new
+  end
+
+  context "for page with og:image" do
+    before :each do
+      stub_request(:get, "www.example.com/og_image.html").to_return(:body => asset_file("og_image.html"))
+      stub_request(:get, "www.example.com/images/kitty.jpg").to_return(:body => asset_file(File.join("images", "kitty.jpg")))
+    end
+
+    describe "#image_to_thumbnail_for_url" do
+      it "should be found og:image" do
+        image_to_thumbnail = @thumbnail_scraper.image_to_thumbnail_for_url("http://www.example.com/og_image.html")
+        image_to_thumbnail.url.to_s.should == "http://www.example.com/images/kitty.jpg"
+      end
+    end
+  end
+
+  context "for page with link img_src" do
+    before :each do
+      stub_request(:get, "www.example.com/img_src.html").to_return(:body => asset_file("img_src.html"))
+      stub_request(:get, "www.example.com/images/kitty.jpg").to_return(:body => asset_file(File.join("images", "kitty.jpg")))
+    end
+
+    describe "#image_to_thumbnail_for_url" do
+      it "should be found link img_src" do
+        image_to_thumbnail = @thumbnail_scraper.image_to_thumbnail_for_url("http://www.example.com/img_src.html")
+        image_to_thumbnail.url.to_s.should == "http://www.example.com/images/kitty.jpg"
+      end
+    end
+  end
+
+  context "for page with images" do
+    before :each do
+      stub_request(:get, "www.example.com/images.html").to_return(:body => asset_file("images.html"))
+      stub_request(:get, "www.example.com/images/kitty.jpg").to_return(:body => asset_file(File.join("images", "kitty.jpg")))
+      stub_request(:get, "www.example.com/images/biggest.jpg").to_return(:body => asset_file(File.join("images", "biggest.jpg")))
+      stub_request(:get, "www.example.com/images/subcatalog/smallest.jpg").to_return(:body => asset_file(File.join("images", "subcatalog", "smallest.jpg")))
+    end
+
+    describe "#image_to_thumbnail_for_url" do
+      it "should be biggest image from page" do
+        image_to_thumbnail = @thumbnail_scraper.image_to_thumbnail_for_url("http://www.example.com/images.html")
+        image_to_thumbnail.url.to_s.should == "http://www.example.com/images/biggest.jpg"
+      end
+    end
+  end
+end

data/spec/units/http_receiver_spec.rb

@@ -0,0 +1,24 @@
+require 'thumbnail_scraper_helper'
+require 'thumbnail_scraper/http_receiver'
+
+module ThumbnailScraper
+  describe HttpReceiver do
+    before :each do
+      @http_receiver = HttpReceiver.new
+    end
+
+    describe "#receive_webpage" do
+      before :each do
+        stub_request(:get, "www.example.com/index.html").to_return(:body => "<p>I'm a webpage</p>")
+      end
+
+      it "should set Webpage#body" do
+        @http_receiver.receive_webpage("http://www.example.com/index.html").body == "<p>I'm a webpage</p>"
+      end
+
+      it "should set Webpage#uri" do
+        @http_receiver.receive_webpage("http://www.example.com/index.html").url.should == URI.parse("http://www.example.com/index.html")
+      end
+    end
+  end
+end

data/spec/units/thumbnail_scraper_spec.rb

@@ -0,0 +1,182 @@
+require 'thumbnail_scraper_helper'
+require 'thumbnail_scraper'
+
+module ThumbnailScraper
+  describe ThumbnailScraper do
+    before :each do
+      @http_receiver = mock("http receiver mock")
+      @thumbnail_scraper = ThumbnailScraper.new(@http_receiver)
+    end
+
+    describe "#image_to_thumbnail_for_url" do
+      context "for webpage with open graph image" do
+        before :each do
+          @page_url = "http://www.example.com/index.html"
+          @image_url = "http://www.example.com/images/image.png"
+          @webpage = mock("webpage")
+          @image = mock("image")
+          @webpage.stub!(:open_graph_image_url).and_return(@image_url)
+          @thumbnail_scraper.stub!(:create_image).with(@image_url).and_return(@image)
+          @webpage.stub!(:has_open_graph_image?).and_return(true)
+          @http_receiver.stub!(:receive_webpage).with(@page_url).and_return(@webpage)
+        end
+
+        it "should return image" do
+          @thumbnail_scraper.image_to_thumbnail_for_url(@page_url).should == @image
+        end
+      end
+
+      context "for webpage with link img_src" do
+        before :each do
+          @page_url = "http://www.example.com/index.html"
+          @image_url = "http://www.example.com/images/image.png"
+          @webpage = mock("webpage")
+          @image = mock("image")
+          @webpage.stub!(:has_open_graph_image?).and_return(false)
+          @webpage.stub!(:linked_image_url).and_return(@image_url)
+          @webpage.stub!(:has_linked_image?).and_return(true)
+          @thumbnail_scraper.stub!(:create_image).with(@image_url).and_return(@image)
+          @http_receiver.stub!(:receive_webpage).with(@page_url).and_return(@webpage)
+        end
+
+        it "should return image" do
+          @thumbnail_scraper.image_to_thumbnail_for_url(@page_url).should == @image
+        end
+      end
+
+      context "for webpage with only images" do
+        before :each do
+          @page_url = "http://www.example.com/index.html"
+          @webpage = mock("webpage")
+          @webpage.stub!(:has_open_graph_image?).and_return(false)
+          @webpage.stub!(:has_linked_image?).and_return(false)
+          @webpage.stub!(:attached_images_urls).and_return([:first, :second, :third])
+          @thumbnail_scraper.stub!(:create_image).and_return(:image)
+          @http_receiver.stub!(:receive_webpage).with(@page_url).and_return(@webpage)
+          @http_receiver.stub!(:receive_image).with(:image).and_return(:image_content)
+          @thumbnail_scraper.stub!(:select_best_possible_image_to_scrap).and_return(:image)
+        end
+
+        it "should use Website#attached_images_urls to get all images" do
+          @webpage.should_receive(:attached_images_urls).and_return([:first, :second, :third])
+          @thumbnail_scraper.image_to_thumbnail_for_url(@page_url)
+        end
+
+        it "should select_best_possible_image_to_scrap" do
+          @thumbnail_scraper.should_receive(:select_best_possible_image_to_scrap).with([:first, :second, :third]).and_return(:image)
+          @thumbnail_scraper.image_to_thumbnail_for_url(@page_url)
+        end
+      end
+    end
+
+    describe "#select_best_possible_image_to_scrap" do
+      before :each do
+        @image1 = mock("image")
+        @image1.stub!(:area).and_return(250)
+        @image2 = mock("image")
+        @image2.stub!(:area).and_return(1000)
+        @images_urls = [:first, :second]
+        @thumbnail_scraper.stub!(:create_image).with(:first).and_return(@image1)
+        @thumbnail_scraper.stub!(:create_image).with(:second).and_return(@image2)
+        @thumbnail_scraper.stub!(:select_valid_images).with([@image1, @image2]).and_return([@image1, @image2])
+      end
+
+      it "should create all images" do
+        @thumbnail_scraper.should_receive(:create_image).with(:first).and_return(@image1)
+        @thumbnail_scraper.should_receive(:create_image).with(:second).and_return(@image2)
+        @thumbnail_scraper.select_best_possible_image_to_scrap(@images_urls)
+      end
+
+      it "should filter out invalid images" do
+        @thumbnail_scraper.should_receive(:select_valid_images).with([@image1, @image2]).and_return([@image1, @image2])
+        @thumbnail_scraper.select_best_possible_image_to_scrap(@images_urls)
+      end
+
+      it "should be biggest (by area) image of valid ones" do
+        @thumbnail_scraper.select_best_possible_image_to_scrap(@images_urls).should == @image2
+      end
+
+      context "for all invalid images" do
+        it "should be nil" do
+          @thumbnail_scraper.should_receive(:select_valid_images).and_return([])
+          @thumbnail_scraper.select_best_possible_image_to_scrap(@images_urls).should be_nil
+        end
+      end
+    end
+
+    describe "#select_valid_images" do
+      before :each do
+        @image1 = mock("image")
+        @image2 = mock("image")
+        @thumbnail_scraper.stub!(:image_is_valid?).with(@image1).and_return(true)
+        @thumbnail_scraper.stub!(:image_is_valid?).with(@image2).and_return(false)
+      end
+
+      it "should use #image_is_valid? to validate image" do
+        @thumbnail_scraper.stub!(:image_is_valid?).with(@image1).and_return(true)
+        @thumbnail_scraper.stub!(:image_is_valid?).with(@image2).and_return(false)
+        @thumbnail_scraper.select_valid_images([@image1, @image2])
+      end
+
+      it "should select only valid images" do
+        @thumbnail_scraper.select_valid_images([@image1, @image2]).should == [@image1]
+      end
+    end
+
+    describe "#image_is_valid?" do
+      before :each do
+        @image = mock("image")
+        @image.stub!(:width).and_return(100)
+        @image.stub!(:height).and_return(100)
+      end
+
+      context "for small image width" do
+        before :each do
+          @image.stub!(:width).and_return(49)
+        end
+
+        it "should be false" do
+          @thumbnail_scraper.image_is_valid?(@image).should be_false
+        end
+      end
+
+      context "for small image height" do
+        before :each do
+          @image.stub!(:height).and_return(49)
+        end
+
+        it "should be false" do
+          @thumbnail_scraper.image_is_valid?(@image).should be_false
+        end
+      end
+
+      context "width to height ratio bigger than 3:1" do
+        before :each do
+          @image.stub!(:width).and_return(301)
+          @image.stub!(:height).and_return(100)
+        end
+
+        it "should be false" do
+          @thumbnail_scraper.image_is_valid?(@image).should be_false
+        end
+      end
+
+      context "height to width ratio bigger than 3:1" do
+        before :each do
+          @image.stub!(:width).and_return(100)
+          @image.stub!(:height).and_return(301)
+        end
+
+        it "should be false" do
+          @thumbnail_scraper.image_is_valid?(@image).should be_false
+        end
+      end
+
+      context "otherwise" do
+        it "should be true" do
+          @thumbnail_scraper.image_is_valid?(@image).should be_true
+        end
+      end
+    end
+  end
+end

data/spec/units/webpage_spec.rb

@@ -0,0 +1,177 @@
+require 'thumbnail_scraper_helper'
+require 'thumbnail_scraper/webpage'
+
+module ThumbnailScraper
+  describe Webpage do
+    before :each do
+      @url = "http://www.example.com/site/site.html"
+      @webpage = Webpage.new(@url, "")
+    end
+
+    describe "#image_url" do
+      context "for relative path" do
+        it "should be with absolute path based on page url and image relative path" do
+          @webpage.image_url("images/kitty.jpg").path.should == "/site/images/kitty.jpg"
+        end
+
+        it "should have same host as page" do
+          @webpage.image_url("images/kitty.jpg").host.should == "www.example.com"
+        end
+      end
+
+      context "for absolute path" do
+        it "should be with absolute path based on image path" do
+          @webpage.image_url("/images/kitty.jpg").path.should == "/images/kitty.jpg"
+        end
+
+        it "should have same host as page" do
+          @webpage.image_url("/images/kitty.jpg").host.should == "www.example.com"
+        end
+      end
+
+      context "for full uri" do
+        it "should be with absolute path based on image path" do
+          @webpage.image_url("http://outersite.com/images/kitty.jpg").path.should == "/images/kitty.jpg"
+        end
+
+        it "should have same host as page" do
+          @webpage.image_url("http://outersite.com/images/kitty.jpg").host.should == "outersite.com"
+        end
+
+        context "for https address" do
+          it "should have image's host" do
+            @webpage.image_url("https://outersite.com/images/kitty.jpg").host.should == "outersite.com"
+          end
+        end
+
+        context "for shortcut address" do
+          it "should have image's host" do
+            @webpage.image_url("//outersite.com/images/kitty.jpg").host.should == "outersite.com"
+          end
+
+          it "should have same scheme as page" do
+            @webpage.image_url("//outersite.com/images/kitty.jpg").scheme.should == "http"
+          end
+        end
+      end
+    end
+
+    describe "#open_graph_image_url" do
+      context "for document with og:image" do
+        before :each do
+          @webpage.body = File.read(asset_file_path("og_image.html"))
+          @webpage.stub!(:image_url).and_return(:image_url)
+        end
+
+        it "should use #image_url to construct final image url" do
+          @webpage.should_receive(:image_url).with("images/kitty.jpg").and_return(:image_url)
+          @webpage.open_graph_image_url
+        end
+
+        it "should return url constructed by image_url" do
+          @webpage.open_graph_image_url.should == :image_url
+        end
+      end
+
+      context "for document without og:image" do
+        before :each do
+          @webpage.body = File.read(asset_file_path("img_src.html"))
+        end
+
+        it "should be nil" do
+          @webpage.open_graph_image_url.should be_nil
+        end
+      end
+    end
+
+    describe "#has_open_graph_image" do
+      context "for nil #open_graph_image_url" do
+        before :each do
+          @webpage.stub!(:open_graph_image_url).and_return(nil)
+        end
+
+        it "should be false" do
+          @webpage.has_open_graph_image?.should be_false
+        end
+      end
+
+      context "for not nil #open_graph_image_url" do
+        before :each do
+          @webpage.stub!(:open_graph_image_url).and_return(:image_url)
+        end
+
+        it "should be true" do
+          @webpage.has_open_graph_image?.should be_true
+        end
+      end
+    end
+
+    describe "#linked_image_url" do
+      context "for document with link img_src" do
+        before :each do
+          @webpage.body = File.read(asset_file_path("img_src.html"))
+          @webpage.stub!(:image_url).and_return(:image_url)
+        end
+
+        it "should use #image_url to construct final image url" do
+          @webpage.should_receive(:image_url).with("images/kitty.jpg").and_return(:image_url)
+          @webpage.linked_image_url
+        end
+
+        it "should return url constructed by image_url" do
+          @webpage.linked_image_url.should == :image_url
+        end
+      end
+
+      context "for document without link img_src" do
+        before :each do
+          @webpage.body = File.read(asset_file_path("og_image.html"))
+        end
+
+        it "should be nil" do
+          @webpage.linked_image_url.should be_nil
+        end
+      end
+    end
+
+    describe "#has_linked_image" do
+      context "for nil #linked_image_url" do
+        before :each do
+          @webpage.stub!(:linked_image_url).and_return(nil)
+        end
+
+        it "should be false" do
+          @webpage.has_linked_image?.should be_false
+        end
+      end
+
+      context "for not nil #linked_image_url" do
+        before :each do
+          @webpage.stub!(:linked_image_url).and_return(:image_url)
+        end
+
+        it "should be true" do
+          @webpage.has_linked_image?.should be_true
+        end
+      end
+    end
+
+    describe "#attached_images_urls" do
+      context "with images in body" do
+        before :each do
+          @webpage.body = File.read(asset_file_path("images.html"))
+          @webpage.stub!(:image_url).with("/images/subcatalog/smallest.jpg").and_return(:smallest_url)
+          @webpage.stub!(:image_url).with("/images/kitty.jpg").and_return(:kitty_url)
+          @webpage.stub!(:image_url).with("images/biggest.jpg").and_return(:biggest_url)
+        end
+
+        it "should return all urls of found images" do
+          urls = @webpage.attached_images_urls
+          urls.should include(:smallest_url)
+          urls.should include(:kitty_url)
+          urls.should include(:biggest_url)
+        end
+      end
+    end
+  end
+end

data/thumbnail_scraper.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/thumbnail_scraper/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jeppe Liisberg", "Jan Filipowski"]
+  gem.email         = ["jachuf@gmail.com"]
+  gem.description   = %q{detect, fetch and generate a thumbnail for any url and store it on s3}
+  gem.summary       = %q{detect, fetch and generate a thumbnail for any url and store it on s3}
+  gem.homepage      = ""
+
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "webmock"
+  gem.add_runtime_dependency "httpclient"
+  gem.add_runtime_dependency "fastimage"
+  gem.add_runtime_dependency "nokogiri"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "thumbnail_scraper"
+  gem.require_paths = ["lib"]
+  gem.version       = ThumbnailScraper::VERSION
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: thumbnail_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jeppe Liisberg
+- Jan Filipowski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fastimage
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: detect, fetch and generate a thumbnail for any url and store it on s3
+email:
+- jachuf@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- lib/thumbnail_scraper.rb
+- lib/thumbnail_scraper/http_receiver.rb
+- lib/thumbnail_scraper/image.rb
+- lib/thumbnail_scraper/version.rb
+- lib/thumbnail_scraper/webpage.rb
+- lib/thumbnail_scraper/with_smart_url.rb
+- spec/sample_pages/images.html
+- spec/sample_pages/images/biggest.jpg
+- spec/sample_pages/images/kitty.jpg
+- spec/sample_pages/images/subcatalog/smallest.jpg
+- spec/sample_pages/img_src.html
+- spec/sample_pages/og_image.html
+- spec/thumbnail_scraper_helper.rb
+- spec/thumbnail_scraper_spec.rb
+- spec/units/http_receiver_spec.rb
+- spec/units/thumbnail_scraper_spec.rb
+- spec/units/webpage_spec.rb
+- thumbnail_scraper.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: detect, fetch and generate a thumbnail for any url and store it on s3
+test_files:
+- spec/sample_pages/images.html
+- spec/sample_pages/images/biggest.jpg
+- spec/sample_pages/images/kitty.jpg
+- spec/sample_pages/images/subcatalog/smallest.jpg
+- spec/sample_pages/img_src.html
+- spec/sample_pages/og_image.html
+- spec/thumbnail_scraper_helper.rb
+- spec/thumbnail_scraper_spec.rb
+- spec/units/http_receiver_spec.rb
+- spec/units/thumbnail_scraper_spec.rb
+- spec/units/webpage_spec.rb