ruby-readability 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93fa92e0a04e5193bacf3e09a77cb8d25ddcff54
-  data.tar.gz: 7d1f0cd78ead23386d15345d0f356c75fe8c6e2a
+  metadata.gz: b9f4f443e32b774c8c2b14856c78e7c593c6ef41
+  data.tar.gz: 3f6916bfc9b1c88c3c45f5e839fe0e2a4b882ab5
 SHA512:
-  metadata.gz: 93b0d6d8fc0bc0800cfd1f03cebb72d61720eaa348996752be1d04b06504130d453d59d6214d3813af050f17f37f40f0b807bc701ef4c3771dc957b45e129a9a
-  data.tar.gz: fb3faf6b214acd44e7f27dda0342f27b25ffe1decc2deeaf2267388d9ab109225aa4959d9099b9f0815f9b014bc24bbe93bbbe8533b94a57fc3ec1b06c8afd6c
+  metadata.gz: fdf2bb73b0ff4db4617c34996e72f23465d33d90a7631eaaa979235fd8f1f8c529dcf39f7930dc447df72e35e640726b0a3567e3cf0abdafb1ab88e46eb4e3ac
+  data.tar.gz: e75ebfeb153e89fbe52e94e0eab2f33865b32c75ed89e5411387d2cfa6a2f92d0671ecc000229d1ac3cf2027d18e7b7050053c32ab44dca05c8f9a35b20a1194

data/lib/readability.rb

@@ -19,7 +19,21 @@ module Readability
       :blacklist                  => nil,
       :whitelist                  => nil
     }.freeze
-
+    
+    REGEXES = {
+        :unlikelyCandidatesRe => /combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i,
+        :okMaybeItsACandidateRe => /and|article|body|column|main|shadow/i,
+        :positiveRe => /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i,
+        :negativeRe => /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i,
+        :divToPElementsRe => /<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i,
+        :replaceBrsRe => /(<br[^>]*>[ \n\r\t]*){2,}/i,
+        :replaceFontsRe => /<(\/?)font[^>]*>/i,
+        :trimRe => /^\s+|\s+$/,
+        :normalizeRe => /\s{2,}/,
+        :killBreaksRe => /(<br\s*\/?>(\s|&nbsp;?)*){1,}/,
+        :videoRe => /http:\/\/(www\.)?(youtube|vimeo)\.com/i
+    }
+    
     attr_accessor :options, :html, :best_candidate, :candidates, :best_candidate_has_image
 
     def initialize(input, options = {})
@@ -129,6 +143,31 @@ module Readability
 
       (list_images.empty? and content != @html) ? images(@html, true) : list_images
     end
+    
+    def images_with_fqdn_uris!(source_uri)
+      images_with_fqdn_uris(@html, source_uri)
+    end
+    
+    def images_with_fqdn_uris(document = @html.dup, source_uri)
+      uri = URI.parse(source_uri)
+      host = uri.host
+      scheme = uri.scheme
+      port = uri.port # defaults to 80
+
+      base = "#{scheme}://#{host}:#{port}/"
+
+      images = []
+      document.css("img").each do |elem|
+        begin
+          elem['src'] = URI.join(base,elem['src']).to_s if URI.parse(elem['src']).host == nil 
+          images << elem['src'].to_s
+        rescue URI::InvalidURIError => exc
+          elem.remove
+        end
+      end
+
+      images(document,true)
+    end
 
     def get_image_size(url)
       w, h = FastImage.size(url)
@@ -144,20 +183,6 @@ module Readability
       image[:width] >= (options[:min_image_width] || 0) && image[:height] >= (options[:min_image_height] || 0)
     end
 
-    REGEXES = {
-        :unlikelyCandidatesRe => /combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i,
-        :okMaybeItsACandidateRe => /and|article|body|column|main|shadow/i,
-        :positiveRe => /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i,
-        :negativeRe => /combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i,
-        :divToPElementsRe => /<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i,
-        :replaceBrsRe => /(<br[^>]*>[ \n\r\t]*){2,}/i,
-        :replaceFontsRe => /<(\/?)font[^>]*>/i,
-        :trimRe => /^\s+|\s+$/,
-        :normalizeRe => /\s{2,}/,
-        :killBreaksRe => /(<br\s*\/?>(\s|&nbsp;?)*){1,}/,
-        :videoRe => /http:\/\/(www\.)?(youtube|vimeo)\.com/i
-    }
-
     def title
       title = @html.css("title").first
       title ? title.text : nil
@@ -444,7 +469,7 @@ module Readability
         weight = class_weight(el)
         content_score = candidates[el] ? candidates[el][:content_score] : 0
         name = el.name.downcase
-
+        
         if weight + content_score < 0
           el.remove
           debug("Conditionally cleaned #{name}##{el[:id]}.#{el[:class]} with weight #{weight} and content score #{content_score} because score + content score was less than zero.")
@@ -452,6 +477,9 @@ module Readability
           counts = %w[p img li a embed input].inject({}) { |m, kind| m[kind] = el.css(kind).length; m }
           counts["li"] -= 100
 
+          # For every img under a noscript tag discount one from the count to avoid double counting
+          counts["img"] -= el.css("noscript").css("img").length
+                
           content_length = el.text.strip.length  # Count the text length excluding any surrounding whitespace
           link_density = get_link_density(el)
 
@@ -465,13 +493,13 @@ module Readability
     end
 
     def clean_conditionally_reason?(name, counts, content_length, options, weight, link_density)
-      if counts["img"] > counts["p"]
+      if (counts["img"] > counts["p"]) && (counts["img"] > 1)
         "too many images"
       elsif counts["li"] > counts["p"] && name != "ul" && name != "ol"
         "more <li>s than <p>s"
       elsif counts["input"] > (counts["p"] / 3).to_i
         "less than 3x <p>s than <input>s"
-      elsif content_length < (options[:min_text_length] || TEXT_LENGTH_THRESHOLD) && (counts["img"] == 0 || counts["img"] > 2)
+      elsif (content_length < options[:min_text_length]) && (counts["img"] != 1)
         "too short a content length without a single image"
       elsif weight < 25 && link_density > 0.2
         "too many links for its weight (#{weight})"

data/ruby-readability.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "ruby-readability"
-  s.version     = '0.6.2'
+  s.version     = '0.7.0'
   s.authors     = ["Andrew Cantino", "starrhorne", "libc", "Kyle Maxwell"]
   s.email       = ["andrew@iterationlabs.com"]
   s.homepage    = "http://github.com/cantino/ruby-readability"
@@ -20,6 +20,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rspec", ">= 2.8"
   s.add_development_dependency "rspec-expectations", ">= 2.8"
   s.add_development_dependency "rr", ">= 1.0"
-  s.add_dependency 'nokogiri', '>= 1.4.2'
+  s.add_dependency 'nokogiri', '>= 1.6.0'
   s.add_dependency 'guess_html_encoding', '>= 0.0.4'
 end

data/spec/readability_spec.rb

@@ -19,6 +19,35 @@ describe Readability do
         </body>
       </html>
     HTML
+    
+    @simple_html_with_img_no_text = <<-HTML
+    <html>
+      <head>
+        <title>title!</title>
+      </head>
+      <body class='main'>
+        <div class="article-img">
+          <img src="http://img.thesun.co.uk/multimedia/archive/01416/dim_1416768a.jpg">
+        </div>
+      </body>
+      </html>
+    HTML
+    
+    @simple_html_with_img_in_noscript = <<-HTML
+    <html>
+      <head>
+        <title>title!</title>
+      </head>
+      <body class='main'>
+        <div class="article-img">
+        <img src="http://img.thesun.co.uk/multimedia/archive/00703/sign_up_emails_682__703711a.gif" width="660" 
+        height="317" alt="test" class="lazy" 
+        data-original="http://img.thesun.co.uk/multimedia/archive/01416/dim_1416768a.jpg">
+        <noscript><img src="http://img.thesun.co.uk/multimedia/archive/01416/dim_1416768a.jpg"></noscript>
+        </div>
+      </body>
+      </html>
+    HTML
   end
 
   describe "images" do
@@ -36,6 +65,7 @@ describe Readability do
                         
       FakeWeb.register_uri(:get, "http://img.thesun.co.uk/multimedia/archive/00703/sign_up_emails_682__703712a.gif",                                            
                            :body => File.read(File.dirname(__FILE__) + "/fixtures/images/sign_up_emails_682__703712a.gif"))
+     
     end
 
     it "should show one image, but outside of the best candidate" do
@@ -115,6 +145,17 @@ describe Readability do
           @doc.images.should == ["http://news.bbcimg.co.uk/media/images/57060000/gif/_57060487_sub_escapes304x416.gif"]
           @doc.best_candidate_has_image.should == true
         end
+        
+        it "should not miss an image if it exists by itself in a div without text" do
+          @doc = Readability::Document.new(@simple_html_with_img_no_text,:tags => %w[div p img a], :attributes => %w[src href], :remove_empty_nodes => false, :do_not_guess_encoding => true)
+          @doc.images.should == ["http://img.thesun.co.uk/multimedia/archive/01416/dim_1416768a.jpg"]
+        end
+        
+        it "should not double count an image between script and noscript" do
+          @doc = Readability::Document.new(@simple_html_with_img_in_noscript,:tags => %w[div p img a], :attributes => %w[src href], :remove_empty_nodes => false, :do_not_guess_encoding => true)
+          @doc.images.should == ["http://img.thesun.co.uk/multimedia/archive/00703/sign_up_emails_682__703711a.gif", "http://img.thesun.co.uk/multimedia/archive/01416/dim_1416768a.jpg"]
+        end
+        
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-readability
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Andrew Cantino
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-18 00:00:00.000000000 Z
+date: 2014-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -61,14 +61,14 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: 1.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: guess_html_encoding
   requirement: !ruby/object:Gem::Requirement