link_thumbnailer 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in link_thumbnailer.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Pierre-Louis Gottfrois
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,127 @@
+# LinkThumbnailer
+
+Ruby gem generating image thumbnails from a given URL. Rank them and give you back an object containing images and website informations. Works like Facebook link previewer.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'link_thumbnailer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install link_thumbnailer
+
+Run:
+
+	$ rails g link_thumbnailer:install
+
+This will add `link_thumbnailer.rb` to `config/initializers/`. See [#Configuration](https://github.com/gottfrois/link_thumbnailer#configuration) for more details.
+
+## Usage
+
+Run `irb` and require mandatory dependencies:
+
+	require 'rails'
+	 => true
+	require 'link_thumbnailer'
+	 => true
+	 
+This gem can handle [Opengraph](http://ogp.me/) protocol. Here is an example with such a website:
+	
+	object = LinkThumbnailer.generate('http://zerply.com')
+	 => #<LinkThumbnailer::Object description="Go beyond the résumé - showcase your work and your talent" image="http://zerply.com/img/front/facebook_icon_green.png" images=["http://zerply.com/img/front/facebook_icon_green.png"] site_name="zerply.com" title="Join Me on Zerply" url="http://zerply.com"> 
+
+	object.title?
+ 	=> true
+ 	object.title
+ 	=> "Join Me on Zerply"
+
+ 	object.url?
+	=> true
+	object.url
+	=> "http://zerply.com"
+
+	object.foo?
+	=> false
+	object.foo
+	=> nil
+
+Now with a regular website with no particular protocol:
+
+	object = LinkThumbnailer.generate('http://foo.com')
+	 => #<LinkThumbnailer::Object description=nil images=[[ JPEG 750x200 750x200+0+0 DirectClass 8-bit 45kb] scene=0] title="Foo.com" url="http://foo.com"> 
+
+	object.title
+	 => "Foo.com"
+	 
+	object.images
+	 => [[ JPEG 750x200 750x200+0+0 DirectClass 8-bit 45kb]
+	scene=0]
+
+	object.images.first.source_url
+	 => #<URI::HTTP:0x007ff7a923ef58 URL:http://foo.com/media/BAhbB1sHOgZmSSItMjAxMi8wNC8yNi8yMC8xMS80OS80MjYvY29yZG92YWJlYWNoLmpwZwY6BkVUWwg6BnA6CnRodW1iSSINNzUweDIwMCMGOwZU/cordovabeach.jpg>
+
+You can check whether this object is valid or not (set mandatory attributes in the initializer, defaults are `[url, title, images]`)
+
+	object.valid?
+ 	=> true
+ 	
+ You also can set `max` and `top` options at runtime:
+ 
+ 	object = LinkThumbnailer.generate('http://foo.com', :top => 10, :max => 20)
+
+## Configuration
+
+In `config/initializers/link_thumbnailer.rb`
+
+	LinkThumbnailer.configure do |config|
+	  # Set mandatory attributes require for the website to be valid.
+	  # You can set `strict` to false if you want to skip this validation.
+	  # config.mandatory_attributes = %w(url title image)
+
+	  # Whether you want to validate given website against mandatory attributes or not.
+	  # config.strict = true
+
+	  # Numbers of redirects before raising an exception when trying to parse given url.
+	  # config.redirect_limit = 3
+
+	  # List of blacklisted urls you want to skip when searching for images.
+	  # config.blacklist_urls = [
+	  #   %r{^http://ad\.doubleclick\.net/},
+	  #   %r{^http://b\.scorecardresearch\.com/},
+	  #   %r{^http://pixel\.quantserve\.com/},
+	  #   %r{^http://s7\.addthis\.com/}
+	  # ]
+
+	  # Fetch 10 images maximum.
+	  # config.max = 10
+
+	  # Return top 5 images only.
+	  # config.top = 5
+	end
+
+## Features
+
+Implemented:
+
+- Implements [OpenGraph](http://ogp.me/) protocol.
+- Find images and sort them according to how well they represent what the page is about (includes absolute images).
+- Sort images based on their size and color.
+- Blacklist some well known advertisings image urls.
+
+Coming soon:
+
+- Cache results on filesystem
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/lib/generators/link_thumbnailer/install_generator.rb

@@ -0,0 +1,15 @@
+module LinkThumbnailer
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+
+      source_root File.expand_path('../../templates', __FILE__)
+
+      desc 'Creates a LinkThumbnailer initializer for your application.'
+
+      def copy_initializer
+        template 'initializer.rb', 'config/initializers/link_thumbnailer.rb'
+      end
+
+    end
+  end
+end

data/lib/generators/templates/initializer.rb

@@ -0,0 +1,26 @@
+# Use this hook to configure LinkThumbnailer bahaviors.
+LinkThumbnailer.configure do |config|
+  # Set mandatory attributes require for the website to be valid.
+  # You can set `strict` to false if you want to skip this validation.
+  # config.mandatory_attributes = %w(url title image)
+
+  # Whether you want to validate given website against mandatory attributes or not.
+  # config.strict = true
+
+  # Numbers of redirects before raising an exception when trying to parse given url.
+  # config.redirect_limit = 3
+
+  # List of blacklisted urls you want to skip when searching for images.
+  # config.blacklist_urls = [
+  #   %r{^http://ad\.doubleclick\.net/},
+  #   %r{^http://b\.scorecardresearch\.com/},
+  #   %r{^http://pixel\.quantserve\.com/},
+  #   %r{^http://s7\.addthis\.com/}
+  # ]
+
+  # Fetch 10 images maximum.
+  # config.max = 10
+
+  # Return top 5 images only.
+  # config.top = 5
+end

data/lib/link_thumbnailer/doc.rb

@@ -0,0 +1,58 @@
+require 'uri'
+
+module LinkThumbnailer
+
+  module Doc
+
+    def doc_base_href
+      base = at('//head/base')
+      base['href'] if base
+    end
+
+    def img_srcs
+      search('//img').map { |i| i['src'] }.compact
+    end
+
+    def img_abs_urls(base_url = nil)
+      result = []
+
+      img_srcs.each do |i|
+        begin
+          u = URI(i)
+        rescue URI::InvalidURIError
+          next
+        end
+
+        result << if u.is_a?(URI::HTTP)
+          u
+        else
+          URI.join(base_url || doc_base_href || source_url, i)
+        end
+      end
+
+      result
+    end
+
+    def title
+      css('title').text.strip
+    end
+
+    def description
+      if element = xpath('//meta[@name="description" and @content]').first
+        return element.attributes['content'].value.strip
+      end
+
+      css('body p').each do |node|
+        if !node.has_attribute?('style') && node.first_element_child.nil?
+          return node.text.strip
+        end
+      end
+
+      nil
+    end
+
+    attr_accessor :source_url
+
+  end
+
+end

data/lib/link_thumbnailer/doc_parser.rb

@@ -0,0 +1,15 @@
+require 'nokogiri'
+
+module LinkThumbnailer
+
+  class DocParser
+
+    def parse(doc_string, source_url = nil)
+      doc = Nokogiri::HTML(doc_string).extend(LinkThumbnailer::Doc)
+      doc.source_url = source_url
+      doc
+    end
+
+  end
+
+end

data/lib/link_thumbnailer/fetcher.rb

@@ -0,0 +1,29 @@
+require 'net/http/persistent'
+
+module LinkThumbnailer
+
+  class Fetcher
+
+    def fetch(url, redirect_count = 0)
+      if redirect_count > LinkThumbnailer.redirect_limit
+        raise ArgumentError,
+          "too many redirects (#{redirect_count})"
+      end
+
+      uri = url.is_a?(URI) ? url : URI(url)
+
+      if uri.is_a?(URI::HTTP)
+        http = Net::HTTP::Persistent.new('linkthumbnailer')
+        http.headers['User-Agent'] = 'linkthumbnailer'
+        resp = http.request(uri)
+        case resp
+          when Net::HTTPSuccess; resp.body
+          when Net::HTTPRedirection; fetch(resp['location'], redirect_count + 1)
+          else resp.error!
+        end
+      end
+    end
+
+  end
+
+end

data/lib/link_thumbnailer/img_comparator.rb

@@ -0,0 +1,18 @@
+module LinkThumbnailer
+
+  module ImgComparator
+
+    def <=> other
+      result = ([other.rows, other.columns].min ** 2) <=>
+        ([rows, columns].min ** 2)
+
+      if result == 0
+        result = other.number_colors <=> number_colors
+      end
+
+      result
+    end
+
+  end
+
+end

data/lib/link_thumbnailer/img_parser.rb

@@ -0,0 +1,47 @@
+require 'RMagick'
+
+module LinkThumbnailer
+
+  class ImgParser
+
+    def initialize(fetcher, img_url_filter)
+      @fetcher = fetcher
+      @img_url_filters = [*img_url_filter]
+    end
+
+    def parse(img_urls)
+      @img_url_filters.each do |filter|
+        img_urls.delete_if { |i| filter.reject?(i) }
+      end
+
+      imgs = []
+      count = 0
+      img_urls.each { |i|
+        break if count >= LinkThumbnailer.max
+        img = parse_one(i)
+        img.extend LinkThumbnailer::ImgComparator
+        imgs << img
+        count += 1
+      }
+
+      imgs.sort!
+
+      imgs.first(LinkThumbnailer.top)
+    end
+
+    def parse_one(img_url)
+      begin
+        img_data = @fetcher.fetch(img_url)
+        img = Magick::ImageList.new.from_blob(img_data).extend(
+          LinkThumbnailer::WebImage
+        )
+        img.source_url = img_url
+        img
+      rescue Exception
+        nil
+      end
+    end
+
+  end
+
+end

data/lib/link_thumbnailer/img_url_filter.rb

@@ -0,0 +1,14 @@
+module LinkThumbnailer
+
+  class ImgUrlFilter
+
+    def reject?(img_url)
+      LinkThumbnailer.blacklist_urls.each do |url|
+        return true if img_url && img_url.to_s[url]
+      end
+      false
+    end
+
+  end
+
+end

data/lib/link_thumbnailer/object.rb

@@ -0,0 +1,24 @@
+require 'hashie'
+
+module LinkThumbnailer
+  class Object < Hashie::Mash
+
+    def method_missing(method_name, *args, &block)
+      method_name = method_name.to_s
+
+      if method_name.end_with?('?')
+        method_name.chop!
+        !self[method_name].nil?
+      else
+        self[method_name]
+      end
+    end
+
+    def valid?
+      return false if self.keys.empty?
+      LinkThumbnailer.mandatory_attributes.each {|a| return false unless self[a] } if LinkThumbnailer.strict
+      true
+    end
+
+  end
+end

data/lib/link_thumbnailer/opengraph.rb

@@ -0,0 +1,18 @@
+module LinkThumbnailer
+
+    class Opengraph
+
+      def self.parse(object, doc)
+        doc.css('meta').each do |m|
+          if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
+            object[$1.gsub('-', '_')] = m.attribute('content').to_s
+          end
+        end
+        object[:images] = [object[:image]] if object[:image]
+
+        object
+      end
+
+    end
+
+end

data/lib/link_thumbnailer/version.rb

@@ -0,0 +1,3 @@
+module LinkThumbnailer
+  VERSION = "0.0.1"
+end

data/lib/link_thumbnailer/web_image.rb

@@ -0,0 +1,10 @@
+module LinkThumbnailer
+
+  module WebImage
+
+    attr_accessor :source_url
+    attr_accessor :doc
+
+  end
+
+end

data/lib/link_thumbnailer.rb

@@ -0,0 +1,76 @@
+# require 'json'
+# require 'nokogiri'
+# require 'link_thumbnailer/parser/opengraph'
+
+require 'link_thumbnailer/object'
+require 'link_thumbnailer/fetcher'
+require 'link_thumbnailer/doc_parser'
+require 'link_thumbnailer/doc'
+require 'link_thumbnailer/img_url_filter'
+require 'link_thumbnailer/img_parser'
+require 'link_thumbnailer/img_comparator'
+require 'link_thumbnailer/web_image'
+
+require 'link_thumbnailer/opengraph'
+
+require 'link_thumbnailer/version'
+
+module LinkThumbnailer
+
+  mattr_accessor :mandatory_attributes
+  @@mandatory_attributes = %w(url title images)
+
+  mattr_accessor :strict
+  @@strict = true
+
+  mattr_accessor :redirect_limit
+  @@redirect_limit = 3
+
+  mattr_accessor :blacklist_urls
+  @@blacklist_urls = [
+    %r{^http://ad\.doubleclick\.net/},
+    %r{^http://b\.scorecardresearch\.com/},
+    %r{^http://pixel\.quantserve\.com/},
+    %r{^http://s7\.addthis\.com/}
+  ]
+
+  mattr_accessor :max
+  @@max = 10
+
+  mattr_accessor :top
+  @@top = 5
+
+  def self.configure
+    yield self
+  end
+
+  def self.generate(url, options = {})
+    @@top = options[:top].to_i if options[:top]
+    @@max = options[:max].to_i if options[:max]
+
+    @object           = LinkThumbnailer::Object.new
+    @fetcher          = LinkThumbnailer::Fetcher.new
+    @doc_parser       = LinkThumbnailer::DocParser.new
+
+    doc_string = @fetcher.fetch(url)
+    doc = @doc_parser.parse(doc_string, url)
+
+    @object[:url] = doc.source_url
+
+    # Try Opengraph first
+    @object = LinkThumbnailer::Opengraph.parse(@object, doc)
+    return @object if @object.valid?
+
+    # Else try manually
+    @img_url_filters  = [LinkThumbnailer::ImgUrlFilter.new]
+    @img_parser       = LinkThumbnailer::ImgParser.new(@fetcher, @img_url_filters)
+
+    @object[:title] = doc.title
+    @object[:description] = doc.description
+    @object[:images] = @img_parser.parse(doc.img_abs_urls.dup)
+
+    return nil unless @object.valid?
+    @object
+  end
+
+end

data/link_thumbnailer.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/link_thumbnailer/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Pierre-Louis Gottfrois"]
+  gem.email         = ["pierrelouis.gottfrois@gmail.com"]
+  gem.description   = %q{Ruby gem generating thumbnail images from a given URL.}
+  gem.summary       = %q{Ruby gem ranking images from a given URL returning an object containing images and website informations.}
+  gem.homepage      = "https://github.com/gottfrois/link_thumbnailer"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "link_thumbnailer"
+  gem.require_paths = ["lib"]
+  gem.version       = LinkThumbnailer::VERSION
+
+  gem.add_dependency(%q{nokogiri}, ['~> 1.4.0'])
+  gem.add_dependency(%q{hashie}, ['~> 1.2.0'])
+  gem.add_dependency(%q{net-http-persistent}, ['> 0'])
+  gem.add_dependency(%q{rmagick}, ['> 0'])
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: link_thumbnailer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Pierre-Louis Gottfrois
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rmagick
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby gem generating thumbnail images from a given URL.
+email:
+- pierrelouis.gottfrois@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/generators/link_thumbnailer/install_generator.rb
+- lib/generators/templates/initializer.rb
+- lib/link_thumbnailer.rb
+- lib/link_thumbnailer/doc.rb
+- lib/link_thumbnailer/doc_parser.rb
+- lib/link_thumbnailer/fetcher.rb
+- lib/link_thumbnailer/img_comparator.rb
+- lib/link_thumbnailer/img_parser.rb
+- lib/link_thumbnailer/img_url_filter.rb
+- lib/link_thumbnailer/object.rb
+- lib/link_thumbnailer/opengraph.rb
+- lib/link_thumbnailer/version.rb
+- lib/link_thumbnailer/web_image.rb
+- link_thumbnailer.gemspec
+homepage: https://github.com/gottfrois/link_thumbnailer
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Ruby gem ranking images from a given URL returning an object containing images
+  and website informations.
+test_files: []