mechanize-content 0.1.1 → 0.1.2

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/lib/mechanize-content.rb

@@ -2,14 +2,12 @@ require 'rubygems'
 require 'mechanize'
 require 'image_size'
 require 'open-uri'
+require 'mechanize-content/util'
 
 class MechanizeContent
   
   attr_accessor :urls
-  
-  MIN_WIDTH  = 64
-  MIN_HEIGHT = 64
-  
+    
   def initialize(*args)
     @urls = *args
   end
@@ -37,7 +35,7 @@ class MechanizeContent
   def fetch_texts
     (@pages || fetch_pages).each do |page|
       text = fetch_text(page)
-      return @best_text = text unless text.nil?
+      return @best_text = text unless text.nil? || text.empty?
     end
     return nil
   end
@@ -126,11 +124,20 @@ class MechanizeContent
       if paragraph.inner_text().length > 10
         readability[paragraph.parent] += 1
       end
+      if !paragraph.parent.attributes.values.nil?
+        if !paragraph.parent.attributes.values.first.nil?
+          if paragraph.parent.attributes.values.first.value.include? "comment"
+            break
+          end
+        end
+      end
       readability[paragraph.parent] += paragraph.inner_text().count(',')
     end
     sorted_results = readability.sort_by { |parent,score| -score }
     if sorted_results.nil? || sorted_results.first.nil?
       return nil
+    elsif !sorted_results.first.first.xpath("//a[@href='http://get.adobe.com/flashplayer/']").empty?
+      return nil
     else
       top_result = sorted_results.first.first
       top_result.css('script').unlink
@@ -140,57 +147,31 @@ class MechanizeContent
       return top_result
     end
   end
-  
-  def get_base_url(doc, url)
-    base_url = doc.xpath("//base/@href").first
-    if base_url.nil?
-      return url
-    else
-      return base_url.value
-    end
-  end
-  
+    
   def fetch_image(page)
     top_content = fetch_content(page)
     if top_content
-      return find_best_image(top_content.css('img'), get_base_url(page.parser, page.uri))
+      return find_best_image(top_content.css('img'), Util.get_base_url(page.parser, page.uri))
     else
       return nil
     end
   end  
-  
-  def valid_image?(width, height, src)
-    if width > MIN_WIDTH && height > MIN_HEIGHT && !src.include?("banner") && !src.include?(".gif")
-      if (!(width == 728) && !(height == 90))
-        return true
-      end
-    end
-    return false
-  end
-  
-  def build_absolute_url(current_src, url)
-    uri = URI.parse(current_src)
-    if uri.relative?
-      current_src = (URI.parse(url.to_s)+current_src).to_s
-    end
-    current_src
-  end
-  
+      
   def find_best_image(all_images, url)
     begin
       current_src = nil
       all_images.each do |img|
         current_src = img["src"]
-        if valid_image?(img['width'].to_i, img['height'].to_i, current_src)
-          return build_absolute_url(current_src, url)
+        if Util.valid_image?(img['width'].to_i, img['height'].to_i, current_src)
+          return Util.build_absolute_url(current_src, url)
         end
       end
       all_images.each do |img|
         current_src = img["src"]
-        current_src = build_absolute_url(current_src, url)
+        current_src = Util.build_absolute_url(current_src, url)
         open(current_src, "rb") do |fh|
           is = ImageSize.new(fh.read)
-          if valid_image?(is.width, is.height, current_src)
+          if Util.valid_image?(is.width, is.height, current_src)
             return current_src
           end
         end

data/lib/mechanize-content/util.rb

@@ -0,0 +1,37 @@
+class MechanizeContent
+
+  class Util
+    
+    MIN_WIDTH  = 64
+    MIN_HEIGHT = 64
+    AD_WIDTH = 728
+    AD_HEIGHT = 90
+    
+    def self.get_base_url(doc, url)
+      base_url = doc.xpath("//base/@href").first
+      if base_url.nil?
+        return url
+      else
+        return base_url.value
+      end
+    end
+    
+    def self.build_absolute_url(current_src, url)
+      if URI.parse(current_src).relative?
+        current_src = (URI.parse(url.to_s)+current_src).to_s
+      end
+      current_src
+    end
+    
+    def self.valid_image?(width, height, src)
+      if width > MIN_WIDTH && height > MIN_HEIGHT && !src.include?("banner") && !src.include?(".gif")
+        if (!(width == AD_WIDTH) && !(height == AD_HEIGHT))
+          return true
+        end
+      end
+      return false
+    end
+    
+  end
+  
+end

data/mechanize-content.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{mechanize-content}
-  s.version = "0.1.1"
+  s.version = "0.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["John Griffin"]
-  s.date = %q{2010-03-19}
+  s.date = %q{2010-03-23}
   s.description = %q{pass in a url or urls and mechanize-content will select the best block of text, image and title by analysing the page content}
   s.email = %q{johnog@gmail.com}
   s.extra_rdoc_files = [
@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
      "Rakefile",
      "VERSION",
      "lib/mechanize-content.rb",
+     "lib/mechanize-content/util.rb",
      "mechanize-content.gemspec",
      "spec/mechanize-content_spec.rb",
      "spec/spec.opts",

data/spec/mechanize-content_spec.rb

@@ -75,32 +75,32 @@ describe "MechanizeContent" do
   
   it "reject all gifs" do
     mc = MechanizeContent.new("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
-    mc.valid_image?(500, 500, "http://www.cmpevents.com/GD10/ablank.gif2").should eql(false)
+    MechanizeContent::Util.valid_image?(500, 500, "http://www.cmpevents.com/GD10/ablank.gif2").should eql(false)
   end
   
   it "reject image with banner in the name" do
     mc = MechanizeContent.new("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
-    mc.valid_image?(500, 500, "http://www.cmpevents.com/GD10/banner.png").should eql(false)
+    MechanizeContent::Util.valid_image?(500, 500, "http://www.cmpevents.com/GD10/banner.png").should eql(false)
   end
   
   it "reject image that is too small" do
     mc = MechanizeContent.new("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
-    mc.valid_image?(64, 500, "http://www.cmpevents.com/GD10/toosmall.png").should eql(false)
+    MechanizeContent::Util.valid_image?(64, 500, "http://www.cmpevents.com/GD10/toosmall.png").should eql(false)
   end
   
   it "allow good images" do
     mc = MechanizeContent.new("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
-    mc.valid_image?(500, 500, "http://www.cmpevents.com/GD10/perfecto.png").should eql(true)
+    MechanizeContent::Util.valid_image?(500, 500, "http://www.cmpevents.com/GD10/perfecto.png").should eql(true)
   end
   
   it "build a base url for images" do
     mc = MechanizeContent.new("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
     page = mc.fetch_page("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
-    mc.get_base_url(page.parser, page.uri).to_s.should eql("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
+    MechanizeContent::Util.get_base_url(page.parser, page.uri).to_s.should eql("https://www.cmpevents.com/GD10/a.asp?option=C&V=11&SessID=10601")
     
     mc = MechanizeContent.new("http://www.mutinydesign.co.uk/scripts/html-base-tag---1/")
     page = mc.fetch_page("http://www.mutinydesign.co.uk/scripts/html-base-tag---1/")
-    mc.get_base_url(page.parser, page.uri).to_s.should eql("http://www.mutinydesign.co.uk/")
+    MechanizeContent::Util.get_base_url(page.parser, page.uri).to_s.should eql("http://www.mutinydesign.co.uk/")
   end
   
   it "find image" do
@@ -146,4 +146,28 @@ describe "MechanizeContent" do
     mc.best_image.should eql("http://assets.vg247.com/current//2010/03/rockbandlogo.jpg")
   end
   
+  it "gog link no text" do
+    mc = MechanizeContent.new("http://www.gog.com/en/gamecard/another_world_15th_anniversary_edition", "http://www.destructoid.com/-nuff-said-good-old-games-gets-another-world-168150.phtml", "http://www.joystiq.com/2010/03/18/another-world-15th-anniversary-edition-now-on-gog-com/")
+    mc.best_title.should eql("Another World: 15th Anniversary Edition - GOG.com")
+    mc.best_text.should eql("Another World -- or Out of this World, as many of you will know it by -- is now on DRM-free digital distribution service Good Old Games. It can be had for $9.99. Need I say more?\rI love the game, even though I have never made it more than oh, five minutes in. It's more or less universally loved by the Destructoid staff. Not long after we got an email detailing the good news, the thread soon reached fifteen or so replies full of praise for the game.\rOther, less exciting recent releases include: Empire Earth II Gold, Gabriel Knight 3, and Aquanox. Not to completely s**t on these games, but this is Another World we're talking about here.")
+    mc.best_image.should eql("http://www.blogcdn.com/www.joystiq.com/media/2010/03/anotherworldheaderimg580px223.jpg")
+  end
+  
+  it "getting wrong blurb from detructoid" do
+    mc = MechanizeContent.new("http://www.destructoid.com/-nuff-said-good-old-games-gets-another-world-168150.phtml")
+    mc.best_title.should eql("Destructoid - 'Nuff said: Good Old Games gets Another World")
+    mc.best_text.should eql("Another World -- or Out of this World, as many of you will know it by -- is now on DRM-free digital distribution service Good Old Games. It can be had for $9.99. Need I say more?\rI love the game, even though I have never made it more than oh, five minutes in. It's more or less universally loved by the Destructoid staff. Not long after we got an email detailing the good news, the thread soon reached fifteen or so replies full of praise for the game.\rOther, less exciting recent releases include: Empire Earth II Gold, Gabriel Knight 3, and Aquanox. Not to completely s**t on these games, but this is Another World we're talking about here.")
+    mc.best_image.should eql(nil)
+  end
+  
+  it "avoid using copy from flash sites" do
+    mc = MechanizeContent.new("http://www.godofwar.com/spartansstandtall/")
+    mc.best_text.should eql(nil)
+  end
+  
+  it "get this flash site to return nil for a title" do
+    mc = MechanizeContent.new("http://www.thewitcher.com/")
+    mc.best_text.should eql(nil)
+  end
+  
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - John Griffin
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-19 00:00:00 +00:00
+date: 2010-03-23 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -62,6 +62,7 @@ files:
 - Rakefile
 - VERSION
 - lib/mechanize-content.rb
+- lib/mechanize-content/util.rb
 - mechanize-content.gemspec
 - spec/mechanize-content_spec.rb
 - spec/spec.opts