mechanize_content 0.2.1 → 0.3.0

data/.rvmrc

@@ -0,0 +1 @@
+rvm 1.9.2@mechanize_content

data/Rakefile

@@ -1,2 +1,7 @@
 require 'bundler'
 Bundler::GemHelper.install_tasks
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -I lib -r mechanize_content.rb"
+end

data/lib/mechanize_content/image.rb

@@ -0,0 +1,61 @@
+module MechanizeContent
+  class Image
+    MIN_WIDTH  = 64
+    MIN_HEIGHT = 64
+    AD_WIDTH = 728
+    AD_HEIGHT = 90
+    
+    def self.best_image(images, base_url)
+      imgs = images.map{|i| Image.new(i, base_url)}
+      top_image = imgs.select{|i| i.interesting_css?}.first || imgs.select{|i| i.interesting_file?}.first
+      top_image.absolute_url if top_image
+    end
+    
+    def initialize(image, base_url)
+      @src      = image["src"]
+      @width    = image["width"].to_i
+      @height   = image["height"].to_i
+      @base_url = base_url
+    end
+    
+    def interesting_css?
+      valid_image?(@width, @height)
+    end
+    
+    def interesting_file?
+      open(absolute_url, "rb") do |fh|
+        is = ImageSize.new(fh.read)
+        return valid_image?(is.width, is.height)
+      end
+    end
+    
+    def valid_image?(width, height)
+      big_enough?(width, height) && not_advertising?(width, height) && allows_hotlinking?
+    end
+    
+    def allows_hotlinking?
+      begin
+        open(absolute_url, "Referer" => "http://splitstate.com")
+      rescue OpenURI::HTTPError, SocketError
+        return false
+      end
+      true
+    end
+    
+    def advertising?(width, height)
+      @src.include?("banner") || @src.include?(".gif") || ((width == AD_WIDTH) && (height == AD_HEIGHT))
+    end
+    
+    def not_advertising?(width, height)
+      !advertising?(width, height)
+    end
+    
+    def big_enough?(width, height)
+      width > MIN_WIDTH && height > MIN_HEIGHT
+    end
+    
+    def absolute_url
+      URI.parse(@src).relative? ? (URI.parse(@base_url.to_s)+@src).to_s : @src
+    end    
+  end
+end

data/lib/mechanize_content/page.rb

@@ -0,0 +1,115 @@
+module MechanizeContent
+  class Page
+    attr_accessor :url
+    
+    def initialize(url)
+      @url = url
+    end
+    
+    def title
+      content.title if content
+    end
+    
+    def text
+      Util.force_utf8(best_content.text) if best_content && best_content.text.size > 50
+    end
+    
+    def image
+      @image ||= best_content ? Image.best_image(images, base_url) : nil
+    end
+    
+    def images
+      best_content.css('img')
+    end
+    
+    def base_url
+      base = content.parser.xpath("//base/@href").first
+      base ? base.value : content.uri
+    end
+        
+    def best_content
+      @best_content ||= find_content
+    end
+    
+    def find_content
+      return nil unless content
+      doc = content.parser
+      readability = {}
+      doc.css('p').each do |paragraph|
+        if readability[paragraph.parent].nil?
+          readability[paragraph.parent] = 0
+        end
+        parent_class = paragraph.parent['class'] || ""
+        parent_id = paragraph.parent['id'] || ""
+        if !parent_class.match('(comment|meta|footer|footnote)').nil?
+          readability[paragraph.parent] -= 50
+        elsif !parent_class.match('((^|\\s)(post|hentry|entry[-]?(content|text|body)?|article[-_]?(content|text|body)?)(\\s|$))').nil?
+          readability[paragraph.parent] += 25
+        end
+
+        if !parent_id.match('(comment|meta|footer|footnote)').nil?
+          readability[paragraph.parent] -= 50
+        elsif !parent_id.match('((^|\\s)(post|hentry|entry[-]?(content|text|body)?|article[-_]?(content|text|body)?)(\\s|$))').nil?
+          readability[paragraph.parent] += 25
+        end
+
+        if paragraph.inner_text().length > 10
+          readability[paragraph.parent] += 1
+        end
+        if !paragraph.parent.attributes.values.nil?
+          if !paragraph.parent.attributes.values.first.nil?
+            if paragraph.parent.attributes.values.first.value.include? "comment"
+              break
+            end
+          end
+        end
+        readability[paragraph.parent] += paragraph.inner_text().count(',')
+      end
+      sorted_results = readability.sort_by { |parent,score| -score }
+      if sorted_results.nil? || sorted_results.first.nil?
+        return nil
+      elsif !sorted_results.first.first.xpath("//a[@href='http://get.adobe.com/flashplayer/']").empty? || !sorted_results.first.first.xpath("//a[@href='http://www.adobe.com/go/getflashplayer']").empty?
+        return nil
+      else
+        top_result = sorted_results.first.first
+        top_result.css('script').unlink
+        top_result.css('iframe').unlink
+        top_result.css('h1').unlink
+        top_result.css('h2').unlink
+        top_result.css("div#date-byline").unlink
+        top_result.css("p.date").unlink
+        top_result.css("div#facebook-like-button").unlink
+        return top_result
+      end
+    end
+    
+    
+    def content
+      @page_content ||= fetch_content
+    end
+    
+    def fetch_content
+      begin
+        page_content = agent.get(@url)
+        page_content if page_content.is_a?(Mechanize::Page)
+      rescue Timeout::Error
+        puts "Timeout - "+@url
+      rescue Errno::ECONNRESET
+        puts "Connection reset by peer - "+@url
+      rescue Mechanize::ResponseCodeError
+        puts "Invalid url"
+      rescue Mechanize::UnsupportedSchemeError
+        puts "Unsupported Scheme"
+      rescue SocketError => e
+        puts e
+      # rescue
+      #   puts "There was a problem connecting - "+@url
+      end
+    end
+    
+    def agent
+      @agent ||= Mechanize.new {|a| a.user_agent_alias = 'Mac Safari'}
+    end
+    
+  end
+end

data/lib/mechanize_content/util.rb

@@ -1,35 +1,8 @@
 module MechanizeContent
-  class Util
-  
-    MIN_WIDTH  = 64
-    MIN_HEIGHT = 64
-    AD_WIDTH = 728
-    AD_HEIGHT = 90
-  
-    def self.get_base_url(doc, url)
-      base_url = doc.xpath("//base/@href").first
-      if base_url.nil?
-        return url
-      else
-        return base_url.value
-      end
+  class Util          
+    def self.force_utf8(string)
+      ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+      ic.iconv(string.delete("\t").delete("\n").strip + ' ')[0..-2]
     end
-  
-    def self.build_absolute_url(current_src, url)
-      if URI.parse(current_src).relative?
-        current_src = (URI.parse(url.to_s)+current_src).to_s
-      end
-      current_src
-    end
-  
-    def self.valid_image?(width, height, src)
-      if width > MIN_WIDTH && height > MIN_HEIGHT && !src.include?("banner") && !src.include?(".gif")
-        if (!(width == AD_WIDTH) && !(height == AD_HEIGHT))
-          return true
-        end
-      end
-      return false
-    end
-  
   end
 end

data/lib/mechanize_content/version.rb

@@ -1,3 +1,3 @@
 module MechanizeContent
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/mechanize_content.rb

@@ -3,187 +3,27 @@ require 'mechanize'
 require 'image_size'
 require 'open-uri'
 require 'mechanize_content/util'
+require 'mechanize_content/page'
+require 'mechanize_content/image'
 
 module MechanizeContent
   class Parser
-
-    attr_accessor :urls
-
+    attr_accessor :pages
+    
     def initialize(*args)
-      @urls = *args.flatten
+      @pages = *args.flatten.map{|url| Page.new(url)}
     end
 
     def best_title
-      @best_title ||= fetch_titles
+      @pages.map{|page| page.title}.compact.first || @pages.first.url
     end
 
     def best_text
-      @best_text ||= fetch_texts
+      @pages.map{|page| page.text}.compact.first
     end
 
     def best_image
-      @best_image ||= fetch_images
-    end
-
-    def fetch_images
-      (@pages || fetch_pages).each do |page|
-        image = fetch_image(page)
-        return image unless image.nil?
-      end
-      return nil
-    end
-
-    def fetch_texts
-      (@pages || fetch_pages).each do |page|
-        text = fetch_text(page)
-        return text unless text.nil? || text.empty?
-      end
-      return nil
-    end
-
-    def fetch_titles
-      (@pages || fetch_pages).each do |page|
-        title = page.title
-        unless title.nil?
-          ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
-          title = ic.iconv(title + ' ')[0..-2]
-          return title
-        end
-
-      end
-      return @urls.first
-    end
-
-    def fetch_pages
-      @pages = []
-      @urls.each do |url|
-        page = fetch_page(url)
-        @pages << page unless page.nil?
-      end
-      @pages
-    end
-
-    def fetch_page(url)
-      begin
-        page = (@agent || init_agent).get(url)
-        if page.class ==  Mechanize::Page
-          return page
-        else
-          return nil
-        end
-      rescue Timeout::Error
-        puts "Timeout - "+url
-      rescue Errno::ECONNRESET
-        puts "Connection reset by peer - "+url
-      rescue Mechanize::ResponseCodeError
-        puts "Invalid url"
-      rescue Mechanize::UnsupportedSchemeError
-        puts "Unsupported Scheme"
-      rescue
-        puts "There was a problem connecting - "+url
-      end
+      @pages.map{|page| page.image}.compact.first
     end
-
-    def init_agent
-      agent = Mechanize.new
-      agent.user_agent_alias = 'Mac Safari'
-      return @agent = agent
-    end
-
-    def fetch_text(page)
-      top_content = fetch_content(page)
-      if top_content
-        text = top_content.text.delete("\t").delete("\n").strip
-        ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
-        text = ic.iconv(text + ' ')[0..-2]
-      else
-        return nil
-      end
-    end
-
-    def fetch_content(page)
-      doc = page.parser
-      readability = {}
-      doc.css('p').each do |paragraph|
-        if readability[paragraph.parent].nil?
-          readability[paragraph.parent] = 0
-        end
-        parent_class = paragraph.parent['class'] || ""
-        parent_id = paragraph.parent['id'] || ""
-        if !parent_class.match('(comment|meta|footer|footnote)').nil?
-          readability[paragraph.parent] -= 50
-        elsif !parent_class.match('((^|\\s)(post|hentry|entry[-]?(content|text|body)?|article[-_]?(content|text|body)?)(\\s|$))').nil?
-          readability[paragraph.parent] += 25
-        end
-
-        if !parent_id.match('(comment|meta|footer|footnote)').nil?
-          readability[paragraph.parent] -= 50
-        elsif !parent_id.match('((^|\\s)(post|hentry|entry[-]?(content|text|body)?|article[-_]?(content|text|body)?)(\\s|$))').nil?
-          readability[paragraph.parent] += 25
-        end
-
-        if paragraph.inner_text().length > 10
-          readability[paragraph.parent] += 1
-        end
-        if !paragraph.parent.attributes.values.nil?
-          if !paragraph.parent.attributes.values.first.nil?
-            if paragraph.parent.attributes.values.first.value.include? "comment"
-              break
-            end
-          end
-        end
-        readability[paragraph.parent] += paragraph.inner_text().count(',')
-      end
-      sorted_results = readability.sort_by { |parent,score| -score }
-      if sorted_results.nil? || sorted_results.first.nil?
-        return nil
-      elsif !sorted_results.first.first.xpath("//a[@href='http://get.adobe.com/flashplayer/']").empty?
-        return nil
-      else
-        top_result = sorted_results.first.first
-        top_result.css('script').unlink
-        top_result.css('iframe').unlink
-        top_result.css('h1').unlink
-        top_result.css('h2').unlink
-        return top_result
-      end
-    end
-
-    def fetch_image(page)
-      top_content = fetch_content(page)
-      if top_content
-        return find_best_image(top_content.css('img'), Util.get_base_url(page.parser, page.uri))
-      else
-        return nil
-      end
-    end  
-
-    def find_best_image(all_images, url)
-      begin
-        current_src = nil
-        all_images.each do |img|
-          current_src = img["src"]
-          if Util.valid_image?(img['width'].to_i, img['height'].to_i, current_src)
-            return Util.build_absolute_url(current_src, url)
-          end
-        end
-        all_images.each do |img|
-          current_src = img["src"]
-          current_src = Util.build_absolute_url(current_src, url)
-          open(current_src, "rb") do |fh|
-            is = ImageSize.new(fh.read)
-            if Util.valid_image?(is.width, is.height, current_src)
-              return current_src
-            end
-          end
-        end
-        return nil
-      rescue Errno::ENOENT
-        puts "No such file - " + current_src
-      rescue 
-        puts "There was a problem connecting - " + current_src
-      end
-    end
-
   end
 end

data/mechanize_content.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.add_dependency("mechanize", "~> 1.0.0")
   s.add_dependency("imagesize", "~> 0.1.1")
-  s.add_development_dependency('rspec', "~> 2.5.0")
+  s.add_development_dependency('rspec', "~> 2.6.0")
+  s.add_development_dependency('vcr', "~> 1.9.0")
   s.add_development_dependency('fakeweb', "~> 1.3.0")
 end