feedme 0.1 → 0.8.0

data/lib/hpricot-util.rb

@@ -0,0 +1,82 @@
+# HTML utils that use hpricot
+# Adapted from code by By Henrik Nyh (http://henrik.nyh.se), Les Hill 
+# (http://blog.leshill.org)
+
+require 'rubygems'
+require 'html-cleaner'
+require 'hpricot'
+require 'active_support'
+
+module FeedMe
+  class HpricotUtil
+    # Like the Rails _truncate_ helper but doesn't break HTML tags or entities.
+    def truncate_html(html, words=15, truncate_string= "...")
+      return if html.nil?
+      doc = Hpricot(html.to_s)
+      doc.inner_text.mb_chars.split.size >= words ? 
+        doc.truncate(words, truncate_string).inner_html : html.to_s
+    end
+
+    # strip all tags from HTML
+    def strip_html(html)
+      (Hpricot.parse(html)/:"text()").to_s
+    end
+
+    # strip tags from HTML and truncate to a certain number of words
+    def strip_truncate_html(input, words=15, truncate_string='...')
+      strip_html(input).split[0..words].join(' ') + truncate_string
+    end
+
+    # sanitize HTML
+    # todo: dup code to fix bugs
+    def clean_html(html)
+      FeedMe::HtmlCleaner.clean(html)  
+    end
+  end
+  
+  @@instance = HpricotUtil.new
+  
+  def FeedMe.html_helper
+    @@instance
+  end
+end
+
+module HpricotTruncator
+  module NodeWithChildren
+    def truncate(words, truncate_string)
+      return self if inner_text.mb_chars.split.size <= words
+      truncated_node = dup
+      truncated_node.name = name
+      truncated_node.raw_attributes = raw_attributes
+      truncated_node.children = []
+      each_child do |node|
+        break if words <= 0
+        node_length = node.inner_text.mb_chars.split.size
+        truncated_node.children << node.truncate(words, truncate_string)
+        words -= node_length
+      end
+      truncated_node
+    end
+  end
+
+  module TextNode
+    def truncate(num_words, truncate_string)
+      words = content.split
+      self.content = (words.size <= num_words ?
+        content : words[0..num_words-1].join(' ') + truncate_string).to_s
+      self
+    end
+  end
+
+  module IgnoredTag
+    def truncate(max_length, ellipsis)
+      self
+    end
+  end
+end
+
+Hpricot::Doc.send(:include,       HpricotTruncator::NodeWithChildren)
+Hpricot::Elem.send(:include,      HpricotTruncator::NodeWithChildren)
+Hpricot::Text.send(:include,      HpricotTruncator::TextNode)
+Hpricot::BogusETag.send(:include, HpricotTruncator::IgnoredTag)
+Hpricot::Comment.send(:include,   HpricotTruncator::IgnoredTag)

data/lib/html-cleaner.rb

@@ -0,0 +1,188 @@
+require 'rubygems'
+require 'hpricot'
+require 'cgi'
+
+module FeedMe
+
+  # Various methods for cleaning up HTML and preparing it for safe public
+  # consumption.
+  #
+  # Documents used for refrence:
+  # - http://www.w3.org/TR/html4/index/attributes.html
+  # - http://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references
+  # - http://feedparser.org/docs/html-sanitization.html
+  # - http://code.whytheluckystiff.net/hpricot/wiki
+  class HtmlCleaner
+
+    # allowed html elements.
+    HTML_ELEMENTS = %w(
+      a abbr acronym address area b bdo big blockquote br button caption center
+      cite code col colgroup dd del dfn dir div dl dt em fieldset font h1 h2 h3
+      h4 h5 h6 hr i img ins kbd label legend li map menu ol optgroup p pre q s
+      samp small span strike strong sub sup table tbody td tfoot th thead tr tt
+      u ul var
+    )
+
+    # allowed attributes.
+    HTML_ATTRS = %w(
+      abbr accept accept-charset accesskey align alt axis border cellpadding
+      cellspacing char charoff charset checked cite class clear cols colspan
+      color compact coords datetime dir disabled for frame headers height href
+      hreflang hspace id ismap label lang longdesc maxlength media method
+      multiple name nohref noshade nowrap readonly rel rev rows rowspan rules
+      scope selected shape size span src start summary tabindex target title
+      type usemap valign value vspace width
+    )
+
+    # allowed attributes, but they can contain URIs, extra caution required.
+    # NOTE: That means this doesnt list *all* URI attrs, just the ones that are allowed.
+    HTML_URI_ATTRS = %w(
+      href src cite usemap longdesc
+    )
+
+    DODGY_URI_SCHEMES = %w(
+      javascript vbscript mocha livescript data
+    )
+
+    class << self
+
+      # Does this:
+      # - Unescape HTML
+      # - Parse HTML into tree
+      # - Find 'body' if present, and extract tree inside that tag, otherwise parse whole tree
+      # - Each tag:
+      #   - remove tag if not whitelisted
+      #   - escape HTML tag contents
+      #   - remove all attributes not on whitelist
+      #   - extra-scrub URI attrs; see dodgy_uri?
+      #
+      # Extra (i.e. unmatched) ending tags and comments are removed.
+      def clean(str)
+        str = unescapeHTML(str)
+
+        doc = Hpricot(str, :fixup_tags => true)
+        doc = subtree(doc, :body)
+
+        # get all the tags in the document
+        # Somewhere near hpricot 0.4.92 "*" starting to return all elements,
+        # including text nodes instead of just tagged elements.
+        tags = (doc/"*").inject([]) { |m,e| m << e.name if(e.respond_to?(:name) && e.name =~ /^\w+$/) ; m }.uniq
+
+        # Remove tags that aren't whitelisted.
+        remove_tags!(doc, tags - HTML_ELEMENTS)
+        remaining_tags = tags & HTML_ELEMENTS
+
+        # Remove attributes that aren't on the whitelist, or are suspicious URLs.
+        (doc/remaining_tags.join(",")).each do |element|
+          next if element.raw_attributes.nil?
+          element.raw_attributes.reject! do |attr,val|
+            !HTML_ATTRS.include?(attr) || (HTML_URI_ATTRS.include?(attr) && dodgy_uri?(val))
+          end
+          element.raw_attributes = element.raw_attributes.build_hash {|a,v| [a, add_entities(v)]}
+        end unless remaining_tags.empty?
+
+        doc.traverse_text {|t| t.set(add_entities(t.to_html))}
+
+        # Return the tree, without comments. Ugly way of removing comments,
+        # but can't see a way to do this in Hpricot yet.
+        doc.to_s.gsub(/<\!--.*?-->/mi, '')
+      end
+
+      # For all other feed elements:
+      # - Unescape HTML.
+      # - Parse HTML into tree (taking 'body' as root, if present)
+      # - Takes text out of each tag, and escapes HTML.
+      # - Returns all text concatenated.
+      def flatten(str)
+        str.gsub!("\n", " ")
+        str = unescapeHTML(str)
+
+        doc = Hpricot(str, :xhtml_strict => true)
+        doc = subtree(doc, :body)
+
+        out = []
+        doc.traverse_text {|t| out << add_entities(t.to_html)}
+
+        return out.join
+      end
+
+      # Returns true if the given string contains a suspicious URL,
+      # i.e. a javascript link.
+      #
+      # This method rejects javascript, vbscript, livescript, mocha and data URLs.
+      # It *could* be refined to only deny dangerous data URLs, however.
+      def dodgy_uri?(uri)
+        uri = uri.to_s
+
+        # special case for poorly-formed entities (missing ';')
+        # if these occur *anywhere* within the string, then throw it out.
+        return true if (uri =~ /&\#(\d+|x[0-9a-f]+)[^;\d]/mi)
+
+        # Try escaping as both HTML or URI encodings, and then trying
+        # each scheme regexp on each
+        [unescapeHTML(uri), CGI.unescape(uri)].each do |unesc_uri|
+          DODGY_URI_SCHEMES.each do |scheme|
+
+            regexp = "#{scheme}:".gsub(/./) do |char|
+              "([\000-\037\177\s]*)#{char}"
+            end
+
+            # regexp looks something like
+            # /\A([\000-\037\177\s]*)j([\000-\037\177\s]*)a([\000-\037\177\s]*)v([\000-\037\177\s]*)a([\000-\037\177\s]*)s([\000-\037\177\s]*)c([\000-\037\177\s]*)r([\000-\037\177\s]*)i([\000-\037\177\s]*)p([\000-\037\177\s]*)t([\000-\037\177\s]*):/mi
+            return true if (unesc_uri =~ %r{\A#{regexp}}mi)
+          end
+        end
+
+        nil
+      end
+
+      # unescapes HTML. If xml is true, also converts XML-only named entities to HTML.
+      def unescapeHTML(str, xml = true)
+        CGI.unescapeHTML(xml ? str.gsub("&apos;", "&#39;") : str)
+      end
+
+      # Adds entities where possible.
+      # Works like CGI.escapeHTML, but will not escape existing entities;
+      # i.e. &#123; will NOT become &amp;#123;
+      #
+      # This method could be improved by adding a whitelist of html entities.
+      def add_entities(str)
+        str.to_s.gsub(/\"/n, '&quot;').gsub(/>/n, '&gt;').gsub(/</n, '&lt;').gsub(/&(?!(\#\d+|\#x([0-9a-f]+)|\w{2,8});)/nmi, '&amp;')
+      end
+
+      private
+
+      # Everything below elment, or the just return the doc if element not present.
+      def subtree(doc, element)
+        doc.at("//#{element}/*") || doc
+      end
+
+      def remove_tags!(doc, tags)
+        (doc/tags.join(",")).remove unless tags.empty?
+      end
+
+    end
+  end
+end
+
+
+module Enumerable #:nodoc:
+  def build_hash
+    result = {}
+    self.each do |elt|
+      key, value = yield elt
+      result[key] = value
+    end
+    result
+  end
+end
+
+# http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/207625
+#  Subject: A simple Hpricot text setter
+#  From: Chris Gehlker <canyonrat mac.com>
+#  Date: Fri, 11 Aug 2006 03:19:13 +0900
+class Hpricot::Text #:nodoc:
+  def set(string)
+    @content = string
+  end
+end

data/lib/nokogiri-util.rb

@@ -0,0 +1,117 @@
+# HTML utils that use nokogiri
+# Adapted from code by Eleo (http://gist.github.com/101410)
+
+require 'rubygems'
+require 'nokogiri'
+require 'sanitize'
+
+module FeedMe
+  class NokogiriUtil
+    # Truncate HTML while preserving tags
+    def truncate_html(text, num_words=15, truncate_string="...")
+      doc = Nokogiri::HTML(html)
+      current = doc.children.first
+	    count = 0
+
+	    while true
+    		# we found a text node
+    		if current.is_a?(Nokogiri::XML::Text)
+    			count += current.text.split.length
+    			# we reached our limit, let's get outta here!
+    			break if count > num_words
+    			previous = current
+    		end
+
+    		if current.children.length > 0
+    			# this node has children, can't be a text node,
+    			# lets descend and look for text nodes
+    			current = current.children.first
+    		elsif !current.next.nil?
+    			#this has no children, but has a sibling, let's check it out
+    			current = current.next
+    		else 
+    			# we are the last child, we need to ascend until we are
+    			# either done or find a sibling to continue on to
+    			n = current
+    			while !n.is_a?(Nokogiri::HTML::Document) and n.parent.next.nil?
+    				n = n.parent
+    			end
+
+    			# we've reached the top and found no more text nodes, break
+    			if n.is_a?(Nokogiri::HTML::Document)
+    				break;
+    			else
+    				current = n.parent.next
+    			end
+    		end
+    	end
+
+    	if count >= num_words
+    	  unless count == num_words
+      		new_content = current.text.split
+  		
+          # If we're here, the last text node we counted eclipsed the number of words
+          # that we want, so we need to cut down on words.  The easiest way to think about
+          # this is that without this node we'd have fewer words than the limit, so all
+          # the previous words plus a limited number of words from this node are needed.
+          # We simply need to figure out how many words are needed and grab that many.
+          # Then we need to -subtract- an index, because the first word would be index zero.
+      
+          # For example, given:
+          # <p>Testing this HTML truncater.</p><p>To see if its working.</p>
+          # Let's say I want 6 words.  The correct returned string would be:
+          # <p>Testing this HTML truncater.</p><p>To see...</p>
+          # All the words in both paragraphs = 9
+          # The last paragraph is the one that breaks the limit.  How many words would we
+          # have without it? 4.  But we want up to 6, so we might as well get that many.
+          # 6 - 4 = 2, so we get 2 words from this node, but words #1-2 are indices #0-1, so
+          # we subtract 1.  If this gives us -1, we want nothing from this node. So go back to
+          # the previous node instead.
+          index = num_words-(count-new_content.length)-1
+          if index >= 0
+            new_content = new_content[0..index]
+      		  current.content = new_content.join(' ') + truncate_string
+    		  else
+    		    current = previous
+    		    current.content = current.content + truncate_string
+    	    end
+    	  end
+
+    		# remove everything else
+    		while !current.is_a?(Nokogiri::HTML::Document)
+    			while !current.next.nil?
+    				current.next.remove
+    			end
+    			current = current.parent
+    		end
+    	end
+
+    	# now we grab the html and not the text.
+    	# we do first because nokogiri adds html and body tags
+    	# which we don't want
+    	doc.root.children.first.inner_html
+    end
+
+    # strip all tags from HTML
+    def strip_html(html)
+      Nokogiri::HTML(html).inner_text
+    end
+
+    # strip tags from HTML and truncate to a certain number of words
+    def strip_truncate_html(html, words=15, truncate_string='...')
+      strip_html(html).split[0..words].join(' ') + truncate_string
+    end
+
+    # sanitize HTML
+    # todo: dup code to fix bugs
+    def clean_html(html)
+      Sanitize.clean(html) 
+    end
+  end
+  
+  @@instance = NokogiriUtil.new
+  
+  def FeedMe.html_helper
+    @@instance
+  end
+end

data/lib/util.rb

@@ -0,0 +1,45 @@
+module FeedMe
+  # Pretty-print an object, with special formatting for hashes
+  # and arrays.
+  def FeedMe.pretty_to_s(obj, indent_step=2, indent=0, code=nil)
+    new_indent = indent + indent_step
+    space = ' ' * indent
+    new_space = ' ' * new_indent
+    str = ''
+    if (obj.is_a?(FeedData) || obj.is_a?(Hash))
+      str << "#{obj.fm_tag_name} " if obj.is_a?(FeedData)
+      str << "{"
+      obj.each_with_index do |item, index|
+        key, value = code.call(*item) if code
+        str << "\n#{new_space}"
+        str << FeedMe.pretty_to_s(key, indent_step, new_indent, code)
+        str << " => " 
+        str << FeedMe.pretty_to_s(value, indent_step, new_indent, code)
+        str << ',' unless index == obj.size-1
+      end
+      str << "\n#{space}}"
+    elsif obj.is_a?(Array)
+      str << "[" 
+      obj.each_with_index do |value, index|
+        str << "\n#{new_space}"
+        str << FeedMe.pretty_to_s(value, indent_step, new_indent, code)
+        str << ',' unless index == obj.size-1
+      end
+      str << "\n#{space}]"
+    elsif obj.is_a? Symbol
+      str << obj.inspect
+    else
+      str << obj.to_s.strip.inspect
+    end
+    return str
+  end
+end
+
+class String
+  def trunc(wordcount, tail='...')
+    words = self.split
+    truncated = words[0..(wordcount-1)].join(' ')
+    truncated += tail if words.size > wordcount
+    truncated
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: feedme
 version: !ruby/object:Gem::Version 
-  version: "0.1"
+  version: 0.8.0
 platform: ruby
 authors: 
 - John Didion
@@ -9,22 +9,13 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-03 00:00:00 -04:00
+date: 2009-12-28 00:00:00 -05:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 2.3.3
-    version: 
+dependencies: []
+
 description: A simple, flexible, and extensible RSS and Atom parser for Ruby. Based on the popular SimpleRSS library, but with many nice extra features.
 email: 
-- jdidion@rubyforge.org
+- code@didion.net
 executables: []
 
 extensions: []
@@ -38,11 +29,16 @@ files:
 - Manifest.txt
 - README.txt
 - Rakefile
-- lib/feedme.rb
 - examples/rocketboom.rb
 - examples/rocketboom.rss
+- lib/feedme.rb
+- lib/hpricot-util.rb
+- lib/html-cleaner.rb
+- lib/nokogiri-util.rb
+- lib/util.rb
+- test/test_helper.rb
 has_rdoc: true
-homepage: http://feedme.rubyforge.org
+homepage: http://wiki.github.com/jdidion/feedme
 licenses: []
 
 post_install_message: