busk-ruby-readability 1.0.6 → 1.0.7

data/lib/readability.rb

@@ -1,35 +1,25 @@
 require 'rubygems'
 require 'nokogiri'
 
+# so to run with non-Rails projects
+class Object
+  def try(method)
+    send method if respond_to? method
+  end
+end
+
 module Readability
   class Document
     TEXT_LENGTH_THRESHOLD = 25
     RETRY_LENGTH = 250
 
-    attr_accessor :options, :html, :best_candidate
+    attr_accessor :document, :base_uri, :request, :options, :best_candidate
 
-    def initialize(input, options = {})
-      @input = input
+    def initialize(document, base_uri, request, options = {})
+      @document = document
+      @base_uri = base_uri
+      @request = request
       @options = options
-      make_html
-    end
-
-    # def charset
-    #   @charset ||= begin
-    #     if content_type = @input.read.to_s.match(/(<meta\s*([^>]*)http-equiv=['"]?content-type['"]?([^>]*))/i)
-    #       if content_type = content_type[0].match(/charset=([\w-]*)/i)
-    #         content_type[1]
-    #       else
-    #         "utf-8"
-    #       end
-    #     else
-    #       "utf-8"
-    #     end
-    #   end
-    # end
-
-    def make_html
-      @html = Nokogiri::HTML(@input, nil, 'UTF-8')
     end
 
     REGEXES = {
@@ -46,9 +36,15 @@ module Readability
         :videoRe => /http:\/\/(www\.)?(youtube|vimeo|ted|player\.vimeo)\.com/i
     }
 
+    # should we get rid of this?
+    def make_html
+      @document.encoding = 'UTF-8'
+      @best_candidate = nil
+    end
+
     def content(remove_unlikely_candidates = true)
-      @html.css("script, style").each {|el| el.remove }
-      @html.search('//comment()').each {|el| el.remove }
+      @document.css("script, style").each {|el| el.remove }
+      @document.search('//comment()').each {|el| el.remove }
 
       article = youtube if is_youtube? && remove_unlikely_candidates
       article = vimeo if is_vimeo? && remove_unlikely_candidates
@@ -75,23 +71,23 @@ module Readability
     end
     
     def is_youtube?
-      (@input.base_uri.to_s =~ /^http:\/\/(www\.)?youtube.com/)
+      (@base_uri.to_s =~ /^http:\/\/(www\.)?youtube.com/)
     end
     
     def is_vimeo?
-      (@input.base_uri.to_s =~ /^http:\/\/(www.)?vimeo.com/)
+      (@base_uri.to_s =~ /^http:\/\/(www.)?vimeo.com/)
     end
 
     def is_ted?
-      (@input.base_uri.to_s =~ /^http:\/\/(www.)?ted.com\/talks/)
+      (@base_uri.to_s =~ /^http:\/\/(www.)?ted.com\/talks/)
     end
     
     def is_special_case?
-      (@input.base_uri.to_s =~ REGEXES[:videoRe])
+      (@base_uri.to_s =~ REGEXES[:videoRe])
     end
     
     def youtube
-      if @input.base_uri.request_uri =~ /\?v=([_\-a-z0-9]+)&?/i
+      if @request =~ /\?v=([_\-a-z0-9]+)&?/i
         Nokogiri::HTML.fragment <<-HTML
           <object width="706" height="422">
             <param name="movie" value="http://www.youtube.com/v/#{$1}?fs=1&amp;hl=en_US"></param>
@@ -106,8 +102,11 @@ module Readability
     end
     
     def vimeo
+      # matches non-channel or pages that used swfobject to print player
+      if @document.css("#clip_id")
+        Nokogiri::HTML.fragment("<iframe src=\"http://player.vimeo.com/video/#{@document.css("#clip_id").attr('value')}\" width=\"572\" height=\"322\" frameborder=\"0\"></iframe>")
       # matches channel pages
-      if (player = @html.css(".player")).present?
+      elsif player = @document.css(".player")
         html = ""
         player.each do |video|
           if video.to_html =~ /clip_id=([0-9]+)/
@@ -115,16 +114,13 @@ module Readability
           end
         end
         Nokogiri::HTML.fragment(html)
-      # matches non-channel or pages that used swfobject to print player
-      elsif @html.to_html =~ /clip_id=([0-9]+)/
-        Nokogiri::HTML.fragment("<iframe src=\"http://player.vimeo.com/video/#{$1}\" width=\"572\" height=\"322\" frameborder=\"0\"></iframe>")
       else
         nil
       end
     end
     
     def ted
-      if (player = @html.css(".copy_paste")).present?
+      if (player = @document.css(".copy_paste")).present?
         unless player.first.attr("value").blank?
           Nokogiri::HTML.fragment(player.first.attr("value").to_s)
         else
@@ -140,8 +136,8 @@ module Readability
       # Things like preambles, content split by ads that we removed, etc.
 
       sibling_score_threshold = [10, best_candidate[:content_score] * 0.2].max
-      output = Nokogiri::XML::Node.new('div', @html)
-      best_candidate[:elem].parent.andand.children.each do |sibling|
+      output = Nokogiri::XML::Node.new('div', @document)
+      best_candidate[:elem].parent.try(:children).each do |sibling|
         append = false
         append = true if sibling == best_candidate[:elem]
         append = true if candidates[sibling] && candidates[sibling][:content_score] >= sibling_score_threshold
@@ -173,10 +169,10 @@ module Readability
 
         debug("Top 5 candidates:")
         sorted_candidates[0...5].each do |candidate|
-          debug("Candidate #{candidate[:elem].andand.name}##{candidate[:elem][:id]}.#{candidate[:elem][:class]} with score #{candidate[:content_score]}")
+          debug("Candidate #{candidate[:elem].try(:name)}##{candidate[:elem][:id]}.#{candidate[:elem][:class]} with score #{candidate[:content_score]}")
         end
 
-        best_candidate = sorted_candidates.first || { :elem => @html.css("body").first, :content_score => 0 }
+        best_candidate = sorted_candidates.first || { :elem => @document.css("body").first, :content_score => 0 }
         #debug("Best candidate #{best_candidate[:elem].andand.name} with score #{best_candidate[:content_score]}")
         best_candidate
       end
@@ -190,7 +186,7 @@ module Readability
 
     def score_paragraphs(min_text_length)
       candidates = {}
-      @html.css("p,td").each do |elem|
+      @document.css("p,td").each do |elem|
         parent_node = elem.parent
         grand_parent_node = parent_node.respond_to?(:parent) ? parent_node.parent : nil
         inner_text = elem.text
@@ -263,7 +259,7 @@ module Readability
     end
 
     def remove_unlikely_candidates!
-      @html.css("*").each do |elem|
+      @document.css("*").each do |elem|
         str = "#{elem[:class]}#{elem[:id]}"
         if str =~ REGEXES[:unlikelyCandidatesRe] && str !~ REGEXES[:okMaybeItsACandidateRe] && elem.name.downcase != 'body'
           debug("Removing unlikely candidate - #{str}")
@@ -273,7 +269,7 @@ module Readability
     end
 
     def transform_misused_divs_into_paragraphs!
-      @html.css("*").each do |elem|
+      @document.css("*").each do |elem|
         if elem.name.downcase == "div"
           # transform <div>s that do not contain other block elements into <p>s
           if elem.inner_html !~ REGEXES[:divToPElementsRe]