plumnailer 0.0.1

data/COPYING

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2010 Matthew M. Boedicker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.textile

@@ -0,0 +1,23 @@
+Compare all of the images on an HTML page and select the one that best
+represents what the page is about (for use as a thumbnail for the entire page).
+
+Project is in an early stage and for now just picks the largest image on the
+page. More sophisticated scoring coming soon.
+
+<pre>
+<code>
+require 'plumnailer'
+
+chooser = Plumnailer::Chooser.new
+# caching fetcher is good for testing
+chooser.fetcher = chooser.img_parser.fetcher =
+  Plumnailer::CachingFetcher.new('/tmp/plumnailer-test')
+
+choice = chooser.choose('http://urls.matthewm.boedicker.org/')
+puts choice.source_url
+
+choice.resize_to_fill(64, 64).write('thumb.jpg')
+</code>
+</pre>
+
+Questions and comments: "matthewm@boedicker.org":mailto:matthewm@boedicker.org

data/Rakefile

@@ -0,0 +1,18 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = 'plumnailer'
+    gemspec.summary = 'Choose the most representative image on an HTML page'
+    gemspec.description = 'Choose the most representative image on an HTML page for use as a thumbnail'
+    gemspec.email = 'matthewm@boedicker.org'
+    gemspec.homepage = 'http://github.com/mmb/plumnailer'
+    gemspec.authors = ['Matthew M. Boedicker']
+
+    %w{
+      nokogiri 1.4.3.1
+      rmagick 2.13.1
+      }.each_slice(2) { |g,v| gemspec.add_dependency(g, ">= #{v}") }
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/plumnailer/caching_fetcher.rb

@@ -0,0 +1,33 @@
+require 'cgi'
+require 'fileutils'
+
+require 'plumnailer'
+
+module Plumnailer
+
+  # Fetch the contents of a url and cache result on filesystem.
+  class CachingFetcher < Fetcher
+
+    def initialize(cache_dir)
+      @cache_dir = cache_dir
+      FileUtils.mkdir_p(cache_dir)
+    end
+
+    # Fetch the contents of a url and cache result on filesystem.
+    def fetch(url)
+      cache_file = File.join(cache_dir, CGI.escape(url.to_s))
+
+      if File.exists?(cache_file)
+        open(cache_file) { |f| f.read }
+      else
+        data = super
+        open(cache_file, 'w') { |f| f.write(data) }
+        data
+      end
+    end
+
+    attr_accessor :cache_dir
+
+  end
+ 
+end

data/lib/plumnailer/chooser.rb

@@ -0,0 +1,45 @@
+module Plumnailer
+
+  # Find the most representative image on a page.
+  class Chooser
+
+    def initialize
+      @fetcher = Plumnailer::Fetcher.new
+      @doc_parser = Plumnailer::DocParser.new
+      @img_url_filters = [Plumnailer::ImgHostnameFilter.new]
+      @img_parser = Plumnailer::ImgParser.new(fetcher)
+      @img_comparator = Plumnailer::ImgComparator
+    end
+
+    # Find the most representative image on a page.
+    def choose(url)
+      doc_string = fetcher.fetch(url)
+
+      doc = doc_parser.parse(doc_string, url)
+
+      img_abs_urls = doc.img_abs_urls.dup
+      img_url_filters.each do |filter|
+        img_abs_urls.delete_if { |i| filter.reject?(i) }
+      end
+
+      imgs = img_parser.parse(img_abs_urls)
+
+      unless imgs.empty?
+        imgs.each do |img|
+          # set source document on image so it can be used in comparator
+          img.doc = doc
+          img.extend(@img_comparator)
+        end
+        imgs.sort.first
+      end
+    end
+
+    attr_accessor :fetcher
+    attr_accessor :doc_parser
+    attr_accessor :img_url_filters
+    attr_accessor :img_parser
+    attr_accessor :img_comparator
+
+  end
+
+end

data/lib/plumnailer/doc.rb

@@ -0,0 +1,32 @@
+require 'uri'
+
+module Plumnailer
+
+  # Nokogiri::HTML:Document mixin.
+  module Doc
+
+    # Return a list of the src attributes of all img tags.
+    def img_srcs
+      search('//img').map { |x| x['src'] }.compact
+    end
+
+    # Return a list of the absolute urls of all imgs in the document.
+    def img_abs_urls(base_url=nil)
+      result = []
+
+      img_srcs.each do |i|
+        begin
+          u = URI(i)
+        rescue URI::InvalidURIError
+          next
+        end
+        result.push(u.is_a?(URI::HTTP) ? u : URI.join(base_url || source_url, i))
+      end
+
+      result
+    end
+
+    attr_accessor :source_url
+  end
+
+end

data/lib/plumnailer/doc_parser.rb

@@ -0,0 +1,19 @@
+require 'nokogiri'
+
+require 'plumnailer'
+
+module Plumnailer
+
+  # Parse an HTML document.
+  class DocParser
+
+    # Parse an HTML document.
+    def parse(doc_string, source_url=nil)
+      doc = Nokogiri::HTML(doc_string).extend(Plumnailer::Doc)
+      doc.source_url = source_url
+      doc
+    end
+
+  end
+ 
+end

data/lib/plumnailer/fetcher.rb

@@ -0,0 +1,16 @@
+require 'open-uri'
+
+module Plumnailer
+
+  # Fetch the contents of a url.
+  class Fetcher
+
+    # Fetch the contents of a url.
+    def fetch(url)
+      uri = url.is_a?(URI) ? url : URI(url)
+      open(uri) { |f| f.read } if uri.is_a?(URI::HTTP)
+    end
+
+  end
+ 
+end

data/lib/plumnailer/img_comparator.rb

@@ -0,0 +1,13 @@
+module Plumnailer
+
+  # Mixin for Magick::Image that adds a comparator for sorting by relevance.
+  module ImgComparator
+
+    def <=> other
+      # can use doc here to take source page into account
+      other.rows * other.columns <=> rows * columns
+    end
+
+  end
+
+end

data/lib/plumnailer/img_hostname_filter.rb

@@ -0,0 +1,21 @@
+module Plumnailer
+
+  # Decide whether to process images based on their url hostname.
+  class ImgHostnameFilter
+
+    # Return true if this image url should not be considered.
+    def reject?(img_url)
+      HostnameRejectPatterns.each do |re|
+        return true if img_url.host and img_url.host[re]
+      end
+      false
+    end
+
+    HostnameRejectPatterns = [
+      %r{^ad\.doubleclick\.net$},
+      %r{^b\.scorecardresearch\.com$},
+      ]
+
+  end
+
+end

data/lib/plumnailer/img_parser.rb

@@ -0,0 +1,39 @@
+require 'RMagick'
+
+require 'plumnailer'
+
+module Plumnailer
+
+  # Fetch image data urls, load into Magick::Image and set some additional
+  # fields.
+  class ImgParser
+
+    def initialize(fetcher)
+      @fetcher = fetcher
+    end
+
+    # Parse image data from one or more urls.
+    def parse(img_urls)
+      if img_urls.respond_to? :map
+        img_urls.map { |i| parse_one i }.compact
+      else
+        parse_one i
+      end
+    end
+
+    # Fetch image data from a url, load into Magick::Image and set some
+    # additional fields.
+    def parse_one(img_url)
+      img_data = fetcher.fetch(img_url)
+      unless !img_data or img_data.empty?
+        img = Magick::ImageList.new.from_blob(img_data).first.extend(
+          Plumnailer::WebImage)
+        img.source_url = img_url
+        img
+      end
+    end
+
+    attr_accessor :fetcher
+  end
+ 
+end

data/lib/plumnailer/web_image.rb

@@ -0,0 +1,9 @@
+module Plumnailer
+
+  # Mixin for Magick::Image that adds url field and source document.
+  module WebImage
+    attr_accessor :source_url
+    attr_accessor :doc
+  end
+
+end

data/lib/plumnailer.rb

@@ -0,0 +1,10 @@
+require 'plumnailer/chooser'
+require 'plumnailer/doc_parser'
+require 'plumnailer/doc'
+require 'plumnailer/fetcher'
+require 'plumnailer/img_comparator'
+require 'plumnailer/img_hostname_filter'
+require 'plumnailer/img_parser'
+require 'plumnailer/web_image'
+
+require 'plumnailer/caching_fetcher'

data/plumnailer.gemspec

@@ -0,0 +1,58 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{plumnailer}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Matthew M. Boedicker"]
+  s.date = %q{2010-11-09}
+  s.description = %q{Choose the most representative image on an HTML page for use as a thumbnail}
+  s.email = %q{matthewm@boedicker.org}
+  s.extra_rdoc_files = [
+    "README.textile"
+  ]
+  s.files = [
+    "COPYING",
+     "README.textile",
+     "Rakefile",
+     "VERSION",
+     "lib/plumnailer.rb",
+     "lib/plumnailer/caching_fetcher.rb",
+     "lib/plumnailer/chooser.rb",
+     "lib/plumnailer/doc.rb",
+     "lib/plumnailer/doc_parser.rb",
+     "lib/plumnailer/fetcher.rb",
+     "lib/plumnailer/img_comparator.rb",
+     "lib/plumnailer/img_hostname_filter.rb",
+     "lib/plumnailer/img_parser.rb",
+     "lib/plumnailer/web_image.rb",
+     "plumnailer.gemspec",
+     "test.rb"
+  ]
+  s.homepage = %q{http://github.com/mmb/plumnailer}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Choose the most representative image on an HTML page}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.3.1"])
+      s.add_runtime_dependency(%q<rmagick>, [">= 2.13.1"])
+    else
+      s.add_dependency(%q<nokogiri>, [">= 1.4.3.1"])
+      s.add_dependency(%q<rmagick>, [">= 2.13.1"])
+    end
+  else
+    s.add_dependency(%q<nokogiri>, [">= 1.4.3.1"])
+    s.add_dependency(%q<rmagick>, [">= 2.13.1"])
+  end
+end
+

data/test.rb

@@ -0,0 +1,36 @@
+$:.unshift(File.join(File.dirname(__FILE__), 'lib'))
+
+require 'sinatra'
+
+require 'plumnailer'
+
+# sinatra web application for testing
+
+get '/' do
+  page_link = img = ''
+  url = params[:url]
+
+  if url
+    chooser = Plumnailer::Chooser.new
+    chooser.fetcher = chooser.img_parser.fetcher =
+      Plumnailer::CachingFetcher.new('/tmp/plumnailer-test')
+    if choice = chooser.choose(url)
+      img_url = choice.source_url
+      page_link = "<p><a href=\"#{url}\">#{url}</a></p>"
+      img = "<p><img src=\"#{img_url}\" /></p>"
+    else
+      img = '<p>No images found</p>'
+    end
+  end
+
+  <<-eos
+<form method="get" action="">
+<input type="text" name="url" size="50" />
+<input type="submit" />
+</form>
+
+#{page_link}
+
+#{img}
+  eos
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification 
+name: plumnailer
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Matthew M. Boedicker
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-09 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 113
+        segments: 
+        - 1
+        - 4
+        - 3
+        - 1
+        version: 1.4.3.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rmagick
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 57
+        segments: 
+        - 2
+        - 13
+        - 1
+        version: 2.13.1
+  type: :runtime
+  version_requirements: *id002
+description: Choose the most representative image on an HTML page for use as a thumbnail
+email: matthewm@boedicker.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.textile
+files: 
+- COPYING
+- README.textile
+- Rakefile
+- VERSION
+- lib/plumnailer.rb
+- lib/plumnailer/caching_fetcher.rb
+- lib/plumnailer/chooser.rb
+- lib/plumnailer/doc.rb
+- lib/plumnailer/doc_parser.rb
+- lib/plumnailer/fetcher.rb
+- lib/plumnailer/img_comparator.rb
+- lib/plumnailer/img_hostname_filter.rb
+- lib/plumnailer/img_parser.rb
+- lib/plumnailer/web_image.rb
+- plumnailer.gemspec
+- test.rb
+has_rdoc: true
+homepage: http://github.com/mmb/plumnailer
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Choose the most representative image on an HTML page
+test_files: []
+