nddrylliog_pismo 0.7.3

data/lib/pismo/external_attributes.rb

@@ -0,0 +1,14 @@
+module Pismo
+  # External attributes return data that comes from external services or programs (e.g. Delicious tags)
+  module ExternalAttributes
+    #include HTTParty
+    #
+    #def delicious_tags
+    #  delicious_info["top_tags"].sort_by { |k, v| v }.reverse.first(5) rescue []
+    #end
+    #
+    #def delicious_info
+    #  @delicious_info ||= self.class.get('http://feeds.delicious.com/v2/json/urlinfo/' + Digest::MD5.hexdigest(@url)).first rescue nil
+    #end
+  end
+end

data/lib/pismo/internal_attributes.rb

@@ -0,0 +1,316 @@
+module Pismo
+  # Internal attributes are different pieces of data we can extract from a document's content
+  module InternalAttributes
+    # Returns the title of the page/content - attempts to strip site name, etc, if possible
+    def title(all = false)
+      # TODO: Memoizations
+      title = @doc.match( 
+                          [
+                            '#pname a',                                                       # Google Code style
+                            '.entryheader h1',                                                # Ruby Inside/Kubrick
+                            '.entry-title a',                                               # Common Blogger/Blogspot rules
+                            '.post-title a',
+                            '.post_title a',
+                            '.posttitle a',
+                            '.post-header h1',
+                            '.entry-title',
+                            '.post-title',
+                            '.post h1',
+                            '.post h3 a',
+                            'a.datitle',          # Slashdot style
+                            '.posttitle',
+                            '.post_title',
+                            '.pageTitle',
+                            '#main h1.title',
+                            '.title h1',                          
+                            '.post h2',
+                            'h2.title',
+                            '.entry h2 a',
+                            '.entry h2',                                                      # Common style
+                            '.boite_titre a',
+                            ['meta[@name="title"]', lambda { |el| el.attr('content') }],
+                            'h1.headermain',
+                            'h1.title',
+                            '.mxb h1',                                                        # BBC News
+                            '#content h1',
+                            '#content h2',
+                            '#content h3',
+                            'a[@rel="bookmark"]',
+                            '.products h2',
+                            '.caption h3',
+                            '#main h2',
+                            '#body h1',
+                            '#wrapper h1',
+                            '#page h1',
+                            '.asset-header h1',
+                            '#body_content h2'
+                          ],
+                          all
+                        )
+      
+      # If all else fails, go to the HTML title
+      if all
+        return [html_title] if !title
+        return ([*title] + [html_title]).uniq
+      else
+        return html_title if !title
+        return title
+      end
+    end
+    
+    def titles
+      title(true)
+    end
+    
+    
+    # HTML title
+    def html_title
+      title = @doc.match('title')
+      return unless title
+      title
+    end
+    
+    # Return an estimate of when the page/content was created
+    # As clients of this library should be doing HTTP retrieval themselves, they can fall to the
+    # Last-Updated HTTP header if they so wish. This method is just rough and based on content only.
+    def datetime
+      # TODO: Clean all this mess up
+      
+      mo = %r{(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec|January|February|March|April|May|June|July|August|September|October|November|December)}i
+      
+      regexen = [
+        /#{mo}\b\s+\d+\D{1,10}\d{4}/i,
+        /(on\s+)?\d+\s+#{mo}\s+\D{1,10}\d+/i,
+        /(on[^\d+]{1,10})\d+(th|st|rd)?.{1,10}#{mo}\b[^\d]{1,10}\d+/i,
+        /\b\d{4}\-\d{2}\-\d{2}\b/i,
+        /\d+(th|st|rd).{1,10}#{mo}\b[^\d]{1,10}\d+/i,
+        /\d+\s+#{mo}\b[^\d]{1,10}\d+/i,
+        /on\s+#{mo}\s+\d+/i,
+        /#{mo}\s+\d+/i,
+        /\d{4}[\.\/\-]\d{2}[\.\/\-]\d{2}/,
+        /\d{2}[\.\/\-]\d{2}[\.\/\-]\d{4}/
+      ]
+      
+      datetime = 10
+      
+      regexen.each do |r|
+        break if datetime = @doc.to_html[r]
+      end
+      
+      return unless datetime && datetime.length > 4
+      
+      # Clean up the string for use by Chronic
+      datetime.strip!
+      datetime.gsub!(/(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)[^\w]*/i, '')
+      datetime.gsub!(/(mon|tues|tue|weds|wed|thurs|thur|thu|fri|sat|sun)[^\w]*/i, '')
+      datetime.sub!(/on\s+/, '')
+      datetime.gsub!(/\,/, '')
+      datetime.sub!(/(\d+)(th|st|rd)/, '\1')
+      
+      Chronic.parse(datetime) || datetime
+    end
+    
+    # Returns the author of the page/content
+    def author(all = false)
+      author = @doc.match([
+                          '.post-author .fn',
+                          '.wire_author',
+                          '.cnnByline b',
+                          '.editorlink',
+                          '.authors p',
+                          ['meta[@name="author"]', lambda { |el| el.attr('content') }],     # Traditional meta tag style
+                          ['meta[@name="Author"]', lambda { |el| el.attr('content') }],     # CNN style
+                          ['meta[@name="AUTHOR"]', lambda { |el| el.attr('content') }],     # CNN style
+                          '.byline a',                                                      # Ruby Inside style
+                          '.byline',
+                          '.post_subheader_left a',                                         # TechCrunch style
+                          '.byl',                                                           # BBC News style
+                          '.articledata .author a',
+                          '#owners a',                                                      # Google Code style
+                          '.author a',
+                          '.author',
+                          '.auth a',
+                          '.auth',
+                          '.cT-storyDetails h5',                                            # smh.com.au - worth dropping maybe..
+                          ['meta[@name="byl"]', lambda { |el| el.attr('content') }],
+                          '.timestamp a',
+                          '.fn a',
+                          '.fn',
+                          '.byline-author',
+                          '.ArticleAuthor a',
+                          '.blog_meta a',
+                          'cite a',
+                          'cite',
+                          '.contributor_details h4 a',
+                          '.meta a'
+                          ], all)
+                          
+      return unless author
+    
+      # Strip off any "By [whoever]" section
+      if String === author
+        author.sub!(/^(post(ed)?\s)?by\W+/i, '')
+        author.tr!('^a-zA-Z 0-9\'', '|')
+        author = author.split(/\|{2,}/).first.to_s
+        author.gsub!(/\s+/, ' ')
+        author.gsub!(/\|/, '')
+        author.strip!
+      elsif Array === author
+        author.map! { |a| a.sub(/^(post(ed)?\s)?by\W+/i, '') }.uniq!
+      end
+      
+      author
+    end
+    
+    def authors
+      author(true)
+    end
+    
+    
+    # Returns the "description" of the page, usually comes from a meta tag
+    def description
+      @doc.match([
+                  ['meta[@name="description"]', lambda { |el| el.attr('content') }],
+                  ['meta[@name="Description"]', lambda { |el| el.attr('content') }],
+                  ['meta[@name="DESCRIPTION"]', lambda { |el| el.attr('content') }],
+                  'rdf:Description[@name="dc:description"]',
+                  '.description'
+       ])
+    end
+    
+    # Returns the "lede(s)" or first paragraph(s) of the story/page
+    def lede(all = false)
+      lede = @doc.match([ 
+                  '.post-text p',
+                  '.post-body p',
+                  '#blogpost p',
+                  '.story-teaser',
+                  '.article .body p',
+                  '//div[@class="entrytext"]//p[string-length()>40]',                      # Ruby Inside / Kubrick style
+                  'section p',
+                  '.entry .text p',
+                  '.hentry .content p',
+                  '.entry-content p',
+                  '#wikicontent p',                                                        # Google Code style
+                  '.wikistyle p',                                                          # GitHub style
+                  '//td[@class="storybody"]/p[string-length()>40]',                        # BBC News style
+                  '//div[@class="entry"]//p[string-length()>100]',
+                  # The below is a horrible, horrible way to pluck out lead paras from crappy Blogspot blogs that
+                  # don't use <p> tags..
+                  ['.entry-content', lambda { |el| el.inner_html[/(#{el.inner_text[0..4].strip}.*?)\<br/, 1] }],
+                  ['.entry', lambda { |el| el.inner_html[/(#{el.inner_text[0..4].strip}.*?)\<br/, 1] }],
+                  '.entry',
+                  '#content p',
+                  '#article p',
+                  '.post-body',
+                  '.entry-content',
+                  '.document_description_short p',    # Scribd
+                  '.single-post p'
+                  ], all)
+
+      # TODO: Improve sentence extraction - this is dire even if it "works for now"
+      if lede && String === lede
+        return (lede[/^(.*?[\.\!\?]\s){1,3}/m] || lede).to_s.strip
+      elsif lede && Array === lede
+        return lede.map { |l| l.to_s[/^(.*?[\.\!\?]\s){1,3}/m].strip || l }.uniq
+      else
+        return reader_doc && !reader_doc.sentences(4).empty? ? reader_doc.sentences(4).join(' ') : nil
+      end
+    end
+    
+    def ledes
+      lede(true) rescue []
+    end
+    
+    # Returns a string containing the first [limit] sentences as determined by the Reader algorithm
+    def sentences(limit = 3)
+      reader_doc && !reader_doc.sentences.empty? ? reader_doc.sentences(limit).join(' ') : nil
+    end
+
+    # Returns any images with absolute URLs in the document
+    def images(limit = 3)
+      reader_doc && !reader_doc.images.empty? ? reader_doc.images(limit) : nil
+    end
+    
+    # Returns the "keywords" in the document (not the meta keywords - they're next to useless now)
+    def keywords(options = {})
+      options = { :stem_at => 20, :word_length_limit => 15, :limit => 20, :remove_stopwords => true, :minimum_score => 2 }.merge(options)
+      
+      words = {}
+      
+      # Convert doc to lowercase, scrub out most HTML tags, then keep track of words
+      cached_title = title.to_s
+      content_to_use = body.to_s.downcase + " " + description.to_s.downcase
+
+      # old regex for safe keeping -- \b[a-z][a-z\+\.\'\+\#\-]*\b
+      content_to_use.downcase.gsub(/\<[^\>]{1,100}\>/, '').gsub(/\.+\s+/, ' ').gsub(/\&\w+\;/, '').scan(/(\b|\s|\A)([a-z0-9][a-z0-9\+\.\'\+\#\-\\]*)(\b|\s|\Z)/i).map{ |ta1| ta1[1] }.compact.each do |word|
+        next if word.length > options[:word_length_limit]
+        word.gsub!(/^[\']/, '')
+        word.gsub!(/[\.\-\']$/, '')
+        next if options[:hints] && !options[:hints].include?(word)
+        words[word] ||= 0
+        words[word] += (cached_title.downcase =~ /\b#{word}\b/ ? 5 : 1)
+      end
+
+      # Stem the words and stop words if necessary
+      d = words.keys.uniq.map { |a| a.length > options[:stem_at] ? a.stem : a }
+      s = Pismo.stopwords.map { |a| a.length > options[:stem_at] ? a.stem : a }
+
+      words.delete_if { |k1, v1| v1 < options[:minimum_score] }
+      words.delete_if { |k1, v1| s.include?(k1) } if options[:remove_stopwords]
+      words.sort_by { |k2, v2| v2 }.reverse.first(options[:limit])
+    end
+    
+    def reader_doc
+      @reader_doc ||= Reader::Document.create(@doc.to_s, @options)
+    end
+    
+    # Returns body text as determined by Reader algorithm
+    def body
+      @body ||= reader_doc.content(true).strip      
+    end
+    
+    # Returns body text as determined by Reader algorithm WITH basic HTML formatting intact
+    def html_body
+      @html_body ||= reader_doc.content.strip      
+    end
+    
+    # Returns URL to the site's favicon
+    def favicon
+      url = @doc.match([['link[@rel="fluid-icon"]', lambda { |el| el.attr('href') }],      # Get a Fluid icon if possible..
+                        ['link[@rel="shortcut icon"]', lambda { |el| el.attr('href') }],
+                        ['link[@rel="icon"]', lambda { |el| el.attr('href') }]])
+      if url && url !~ /^http/ && @url
+        url = URI.join(@url , url).to_s
+      end
+      
+      url
+    end
+    
+    # Returns URL(s) of Web feed(s)
+    def feed(all = false)
+      url = @doc.match([['link[@type="application/rss+xml"]', lambda { |el| el.attr('href') }],
+                        ['link[@type="application/atom+xml"]', lambda { |el| el.attr('href') }]], all
+      )
+      
+      if url && String === url && url !~ /^http/ && @url
+        url = URI.join(@url , url).to_s
+      elsif url && Array === url
+        url.map! do |u|
+          if u !~ /^http/ && @url
+            URI.join(@url, u).to_s
+          else
+            u
+          end
+        end
+        url.uniq!
+      end
+      
+      url
+    end
+    
+    def feeds
+      feed(true)
+    end
+  end
+end

data/lib/pismo/reader.rb

@@ -0,0 +1,19 @@
+module Pismo
+  module Reader
+    class Document
+
+      def self.create(raw_content, options = {})
+        type = options.delete(:reader)
+        case type
+        when :score
+          Pismo::Reader::Tree.new(raw_content, options)
+        when :cluster
+          Pismo::Reader::Cluster.new(raw_content, options)
+        else
+          Pismo::Reader::Tree.new(raw_content, options)
+        end
+      end
+
+    end  
+  end
+end

data/lib/pismo/reader/base.rb

@@ -0,0 +1,259 @@
+require 'nokogiri'
+require 'sanitize'
+begin; require 'ap'; rescue LoadError; end
+
+module Pismo
+  module Reader
+    class Base
+      attr_reader :raw_content, :doc, :content_candidates, :options
+      
+      # Elements to keep for /input/ sanitization
+      OK_ELEMENTS = %w{a td br th tbody table tr div span img strong em b i body html head title p h1 h2 h3 h4 h5 h6 pre code tt ul li ol blockquote font big small section article abbr audio video cite dd dt figure caption sup form dl dt dd center}
+  
+      # Build a tree of attributes that are allowed for each element.. doing it this messy way due to how Sanitize works, alas
+      OK_ATTRIBUTES = {}
+      OK_CLEAN_ATTRIBUTES = {}
+      OK_ELEMENTS.each { |el| OK_ATTRIBUTES[el] = %w{id class href name content type alt title src} }
+      OK_ELEMENTS.each { |el| OK_CLEAN_ATTRIBUTES[el] = %w{href title src alt} }
+      
+      
+      # Words that we'd like to see in class and ID names for "content"
+      GOOD_WORDS = %w{content post blogpost main story body entry text desc asset hentry single entrytext postcontent bodycontent}.uniq
+  
+      # Words that indicate crap in general
+      BAD_WORDS = %w{reply metadata options commenting comments comment about footer header outer credit sidebar widget subscribe clearfix date social bookmarks links share video watch excerpt related supplement accessibility offscreen meta title signup blq secondary feedback featured clearfix small job jobs listing listings navigation nav byline addcomment postcomment trackback neighbor ads commentform fbfans login similar thumb link blogroll grid twitter wrapper container nav sitesub printfooter editsection visualclear catlinks hidden toc contentsub caption disqus rss shoutbox sponsor blogcomments}.uniq
+      
+      # Words that kill a branch dead
+      FATAL_WORDS = %w{comments comment bookmarks social links ads related similar footer digg totop metadata sitesub nav sidebar commenting options addcomment leaderboard offscreen job prevlink prevnext navigation reply-link hide hidden sidebox archives vcard}
+      
+      META_WORDS = %w{january february march april may june july august september october november december jan feb mar apr may jun jul aug sep oct nov dec st th rd nd comments written posted on at published 2000 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 updated last gmt est pst pdt edt cet cdt cst article feature featured filed under comment comments follow twitter facebook email e-mail register story continue continues reading read inside more page next related response responses respond contact street phone tel email e-mail fax info tags tagged tag thanks credit creative commons copy nbsp lt gt this friend printable version subscribe rss mail follow twitter article via leave}.uniq
+  
+      WONT_CONTAIN_FULL_CONTENT = %w{h1 h2 h3 h4 h5 h6 h6 li ol ul br a img meta cite strong em i b input head small big code title sup sub dd dt}
+      COULD_CONTAIN_FULL_CONTENT = %w{body div p table tr td article pre blockquote tbody section}
+  
+      ## Output sanitization element sets
+      BLOCK_OUTPUT_ELEMENTS = %w{div p h2 h3 h4 h5 h6 li dl pre ul ol blockquote section article audio video cite dd dt figure caption br table tr td thead tbody tfoot}
+      INLINE_OUTPUT_ELEMENTS = %w{a img b strong em i br code sup font small big dd dt}
+      OUTPUT_ELEMENTS = BLOCK_OUTPUT_ELEMENTS + INLINE_OUTPUT_ELEMENTS
+      NON_HEADER_ELEMENTS = %w{p br}
+
+      # Create a document object based on the raw HTML content provided
+      def initialize(raw_content, options = {})
+        @options = options
+        @raw_content = Pismo::Document.clean_html(raw_content)
+        build_doc
+      end
+
+      def build_doc
+        @content = {}
+        
+        if RUBY_VERSION > "1.9"
+          @raw_content.encode!("UTF-8", :invalid => :replace, :replace => '?') if @raw_content.encoding != "UTF-8"
+          @raw_content.encode!("ASCII-8BIT", :invalid => :replace, :replace => '?') if !@raw_content.valid_encoding?
+        end
+  
+        # Normalize whitespace (as much to make debugging sessions look nice as anything else)
+        @raw_content.gsub!(/\s{2,}/, ' ')
+        @raw_content.gsub!(/\r/, "\n")
+        @raw_content.gsub!(/\n{3,}/, "\n\n")
+        @raw_content.gsub!(/(\<br(\s\/)?\>){2,}/, "</p><p>")
+        
+        # Remove scripts manually, Sanitize and/or Nokogiri seem to go a bit funny with them
+        @raw_content.gsub!(/\<script .*?\<\/script\>/im, '')
+        
+        # Get rid of bullshit "smart" quotes and other Unicode nonsense
+        @raw_content.force_encoding("ASCII-8BIT") if RUBY_VERSION > "1.9"
+        @raw_content.gsub!("\xe2\x80\x89", " ")
+        @raw_content.gsub!("\xe2\x80\x99", "'")
+        @raw_content.gsub!("\xe2\x80\x98", "'")
+        @raw_content.gsub!("\xe2\x80\x9c", '"')
+        @raw_content.gsub!("\xe2\x80\x9d", '"')
+        @raw_content.gsub!("\xe2\x80\xf6", '.')
+        @raw_content.force_encoding("UTF-8") if RUBY_VERSION > "1.9"
+        
+              
+        # Sanitize the HTML
+        @raw_content = Sanitize.clean(@raw_content,
+          :elements => OK_ELEMENTS,
+          :attributes => OK_ATTRIBUTES,
+          :remove_contents => true,
+          :output_encoding => 'utf-8'
+        )
+              
+        @doc = Nokogiri::HTML(@raw_content, nil, 'utf-8')
+
+        # Do a pre clean up of elements. 
+        @doc.css("div, span, table, tr, td, pre").each do |el|
+          # Any block elements with no child block elements can become paragraphs
+          if (BLOCK_OUTPUT_ELEMENTS & el.inner_html.scan(/\<(\w+)/).flatten).empty?
+            el.name = "p"
+          elsif el.name != "span"
+            el.name = "div"
+          end
+
+          # Any SPANs that aren't within paragraphs can become paragraphs too
+          el.name = "p" if el.name == "span" && !el.path.scan(/[a-z]+/).include?('p')
+
+          el.remove if (FATAL_WORDS & (el['id'].to_s + ' ' + el['class'].to_s).downcase.strip.scan(/[a-z]+/)).size > 0
+        end
+        
+        analyze
+      end
+    
+      # Return the content from best match number of index (default 0) and, optionally, clean it to plain-text
+      def content(clean = false, index = 0)
+        return @content[[clean, index]] if @content[[clean, index]]
+        return '' if !@content_candidates || @content_candidates.empty?
+        
+        content_branch = content_at(index)
+        orphans_to_remove = []
+        
+        #ap content_branch.to_html
+        #exit
+        
+        # Go through every piece of the content and rip out sections that contain too many tags compared to words
+        # This is usually indicative of "widgets" or link bar sections
+        content_branch.css('*').each_with_index do |el, i|
+          next unless el
+
+          if el.name == "h1"
+            el.remove
+            next
+          end
+
+          if el.name == "h2" && content_branch.inner_html.scan('<h2').size == 1
+            el.remove
+          end
+
+          # Remove elements that contain words but there are more tags than words overall
+          # First, count the words
+          #word_count = 0
+          #el.traverse do |subel|
+          #  if subel.text? && subel.path !~ /\/a\// && subel.path !~ /\/(h1|h2|h3|h4|h5|h6)\//
+          #    word_count += (subel.text.downcase.scan(/[a-z]{4,}/) - META_WORDS).size
+          #  end
+          #end
+          #
+          ## .. then count the tags
+          #
+          #inner_tags = el.inner_html.scan(/\<\w.*?\>/).size
+          #if word_count < inner_tags && inner_tags > 3 && word_count < 250
+          #  puts "At #{el.name} #{el['id']} #{el['class']} containing '#{el.text[0..20]}' we have #{word_count} valid words to #{el.inner_html.scan(/\<\w.*?\>/).size} tags"
+          #  #puts "Removing #{el.name} #{el['id']} #{el['class']} TOO MANY TAGS FOR WORDS"
+          #  el.remove
+          #  next
+          #end
+
+          # If there are at least 2 words and a third of them are "meta words," remove the element
+          #inner_words = el.text.to_s.downcase.scan(/[a-z]{3,}/)
+          #if BLOCK_OUTPUT_ELEMENTS.include?(el.name) && inner_words.size >= 2
+          #  if ((inner_words & META_WORDS).size >= (inner_words.size / 3))
+          #    el.remove
+          #  end
+          #end
+
+          if el.text && el.text.strip.length < 3 && !%w{img}.include?(el.name) && el.inner_html !~ /\<img/
+            el.remove
+            next
+          end
+
+          if el.name == "p" && el.text !~ /(\.|\?|\!|\"|\')(\s|$)/ && el.inner_html !~ /\<img/
+            el.remove
+            next
+          end
+
+          # If the ID or class of the element contains a fatally bad word, get rid of it
+          if (BAD_WORDS & (el['id'].to_s + ' ' + el['class'].to_s).downcase.scan(/[a-z]+/)).length > 0
+            #puts "Removing #{el.name} #{el['id']} #{el['class']} BAD"
+            el.remove
+            next
+          end
+        end
+
+        # If a title was found early in the result document but had text before it, remove that text - it's probably crap
+        orphans_to_remove.each { |el| el.remove }
+        
+        # Clean up the HTML again - Nokogiri outputs it with full doctype and crap
+        clean_html = strip(Sanitize.clean(content_branch.to_html, :elements => (clean ? BLOCK_OUTPUT_ELEMENTS : OUTPUT_ELEMENTS), :attributes => (clean ? OK_CLEAN_ATTRIBUTES : OK_ATTRIBUTES)))
+        
+        # If the content is desired as "clean" (i.e. plain-text), do some quick fix-ups
+        if clean
+          # Get rid of line break tags, make list items look nice, remove all other HTML tags, and clean up spaces and newlines
+          clean_html.gsub!(/<br.*?>/, "\n")
+          clean_html.gsub!(/<li>/, '* ')
+          clean_html.gsub!(/<\w+>/, '')
+          clean_html.gsub!(/<\/\w+>/, "\n")
+          clean_html.gsub!(/\ +/, ' ')
+          clean_html.gsub!(/^\s+\n/, "\n")
+          clean_html.gsub!(/\n{2,}/, "\n")
+          clean_html.strip!
+        end
+        
+        # If tags butt up against each other across lines, remove the line break(s)
+        clean_html.gsub!(/\>\n+\</, '><')
+
+        # Get rid of images whose sources are relative (TODO: Make this optional)
+        clean_html.gsub!(/\<img .*?\>/i) do |img_tag|
+          img_tag =~ /\Whttp/ ? img_tag : ''
+        end
+
+        # Remove empty tags
+        clean_html.gsub!(/<(\w+)><\/\1>/, "")
+
+        # Just a messy, hacky way to make output look nicer with subsequent paragraphs..
+        clean_html.gsub!(/<\/(div|p|h1|h2|h3|h4|h5|h6)>/, '</\1>' + "\n\n")
+        
+        @content[[clean, index]] = clean_html
+      end
+          
+      def sentences(qty = 3)
+        clean_content = Sanitize.clean(content, :elements => NON_HEADER_ELEMENTS, :attributes => OK_CLEAN_ATTRIBUTES, :remove_contents => %w{h1 h2 h3 h4 h5 h6})
+
+        fodder = ''
+        doc = Nokogiri::HTML(clean_content, nil, 'utf-8')
+  
+        doc.traverse do |el|
+          path_segments = el.path.scan(/[a-z]+/)[2..-1]
+          next unless path_segments && path_segments.length > 1
+          if el.text? && el.text.strip.length < 3
+            el.remove
+            next
+          end
+          if el.text? && NON_HEADER_ELEMENTS.include?(path_segments[-2]) 
+            text = el.text.strip
+            text += "." if text !~ /[\.\!\?\"\']$/
+            fodder += text + "\n" 
+          end
+        end
+        
+        fodder = content(true) if fodder.to_s.length < 50
+        fodder.gsub!(/\b\w\W\s/, '')
+        
+        #sentences = fodder.scan(/([\&\w\s\-\'\,\+\.\/\\\:\#\(\)\=\"\?\!]+?[\.\?\!])(\s|\Z)/im).map { |s| s.first }
+        sentences = fodder.scan(/(.+?[\.\?\!])(\s|\Z)/im).map { |s| s.first.strip }
+        
+        sentences.compact!
+        sentences.map! { |s| s.strip }
+        sentences.map! { |s| s.sub(/^[^\"\'a-z0-9\(\[]+/im, '') }
+        sentences.map! { |s| s.sub(/[^a-z0-9\'\"\)\]\.\!\:\?]+$/im, '') }
+        sentences.map! { |s| s.gsub(/\s+/m, ' ') }
+        sentences.first(qty)
+      end
+      
+      def images(qty = 3)
+        doc = Nokogiri::HTML(content, nil, 'utf-8')
+        images = []
+        doc.css("img").each do |img|
+          images << img['src']
+          break if images.length == qty
+        end
+        images
+      end
+      
+      # Remove leading and trailing spaces on lines throughout a string (a bit like String#strip, but for multi-lines)
+      def strip(s)
+        s.gsub(/^\s+/, '').gsub(/\s+$/, '')
+      end
+    end  
+  end
+end