feedtools 0.2.24 → 0.2.25

data/lib/feed_tools/feed_item.rb

@@ -1273,6 +1273,21 @@ module FeedTools
               ], :select_result_value => true)
             )
           end
+          if @author.name.blank? && !@author.raw.blank? &&
+              !@author.email.blank?
+            name_scan = @author.raw.scan(
+              /"?([^"]*)"? ?[\(<].*#{@author.email}.*[\)>].*/)
+            if name_scan.flatten.size == 1
+              @author.name = name_scan.flatten[0].strip
+            end
+            if @author.name.blank?
+              name_scan = @author.raw.scan(
+                /.*#{@author.email} ?[\(<]"?([^"]*)"?[\)>].*/)
+              if name_scan.flatten.size == 1
+                @author.name = name_scan.flatten[0].strip
+              end
+            end
+          end
           @author.name = nil if @author.name.blank?
           @author.raw = nil if @author.raw.blank?
           @author.email = nil if @author.email.blank?
@@ -1839,11 +1854,15 @@ module FeedTools
             xml_builder.title(FeedTools::HtmlHelper.strip_html_tags(self.title))
           end
           unless self.link.blank?
-            xml_builder.link(link)
+            xml_builder.link(self.link)
           end
           unless self.author.nil? || self.author.name.nil?
             xml_builder.tag!("dc:creator", self.author.name)
           end
+          unless self.author.nil? || self.author.email.nil? ||
+              self.author.name.nil?
+            xml_builder.author("#{self.author.email} (#{self.author.name})")
+          end
           unless self.summary.blank?
             xml_builder.description(self.summary)
           end

data/lib/feed_tools/helpers/retrieval_helper.rb

@@ -132,7 +132,9 @@ module FeedTools
           options[:form_data] = {} if options[:form_data].blank?
           request_params << options[:form_data]
         end
+        Thread.pass
         response = http.send(http_operation, *request_params)
+        Thread.pass
         
         case response
         when Net::HTTPSuccess
@@ -167,7 +169,7 @@ module FeedTools
 
           redirected_location = response['location']
           redirected_location = FeedTools::UriHelper.resolve_relative_uri(
-            redirected_location, [uri.host])
+            redirected_location, [uri.to_s])
           
           if options[:response_chain].assoc(redirected_location) != nil
             raise FeedAccessError,
@@ -191,7 +193,7 @@ module FeedTools
         end
       rescue SocketError
         raise FeedAccessError, 'Socket error prevented feed retrieval'
-      rescue Timeout::Error
+      rescue Timeout::Error, Errno::ETIMEDOUT
         raise FeedAccessError, 'Timeout while attempting to retrieve feed'
       rescue Errno::ENETUNREACH
         raise FeedAccessError, 'Network was unreachable'

data/lib/feed_tools/helpers/uri_helper.rb

@@ -23,11 +23,12 @@
 
 require 'feed_tools'
 require 'uri'
-
+  
 module FeedTools
   # Generic url processing methods needed in numerous places throughout
   # FeedTools
   module UriHelper
+    
     # Returns true if the idn module can be used.
     def self.idn_enabled?
       # This is an override variable to keep idn from being used even if it

data/lib/feed_tools/monkey_patch.rb

@@ -0,0 +1,625 @@
+require 'rexml/document'
+
+module REXML # :nodoc:
+  class LiberalXPathParser < XPathParser # :nodoc:
+    
+  private
+  
+    # Monkey Patch for Ruby 1.8.4
+    def expr( path_stack, nodeset, context=nil )
+      #puts "#"*15
+      #puts "In expr with #{path_stack.inspect}"
+      #puts "Returning" if path_stack.length == 0 || nodeset.length == 0
+      node_types = ELEMENTS
+      return nodeset if path_stack.length == 0 || nodeset.length == 0
+      while path_stack.length > 0
+        #puts "Path stack = #{path_stack.inspect}"
+        #puts "Nodeset is #{nodeset.inspect}"
+        case (op = path_stack.shift)
+        when :document
+          nodeset = [ nodeset[0].root_node ]
+          #puts ":document, nodeset = #{nodeset.inspect}"
+
+        when :qname
+          #puts "IN QNAME"
+          prefix = path_stack.shift
+          name = path_stack.shift
+          ns = @namespaces[prefix]
+          ns = ns ? ns : ''
+          nodeset.delete_if do |node|
+            # FIXME: This DOUBLES the time XPath searches take
+            ns = node.namespace( prefix ) if node.node_type == :element and ns == ''
+            #puts "NS = #{ns.inspect}"
+            #puts "node.node_type == :element => #{node.node_type == :element}"
+            if node.node_type == :element
+              #puts "node.name == #{name} => #{node.name == name}"
+              if node.name.downcase == name.downcase
+                #puts "node.namespace == #{ns.inspect} => #{node.namespace == ns}"
+              end
+            end
+            !(node.node_type == :element and 
+              node.name.downcase == name.downcase and 
+              node.namespace == ns )
+          end
+          node_types = ELEMENTS
+
+        when :any
+          #puts "ANY 1: nodeset = #{nodeset.inspect}"
+          #puts "ANY 1: node_types = #{node_types.inspect}"
+          nodeset.delete_if { |node| !node_types.include?(node.node_type) }
+          #puts "ANY 2: nodeset = #{nodeset.inspect}"
+
+        when :self
+          # This space left intentionally blank
+
+        when :processing_instruction
+          target = path_stack.shift
+          nodeset.delete_if do |node|
+            (node.node_type != :processing_instruction) or 
+            ( target!='' and ( node.target != target ) )
+          end
+
+        when :text
+          nodeset.delete_if { |node| node.node_type != :text }
+
+        when :comment
+          nodeset.delete_if { |node| node.node_type != :comment }
+
+        when :node
+          # This space left intentionally blank
+          node_types = ALL
+
+        when :child
+          new_nodeset = []
+          nt = nil
+          for node in nodeset
+            nt = node.node_type
+            new_nodeset += node.children if nt == :element or nt == :document
+          end
+          nodeset = new_nodeset
+          node_types = ELEMENTS
+
+        when :literal
+          literal = path_stack.shift
+          if literal =~ /^\d+(\.\d+)?$/
+            return ($1 ? literal.to_f : literal.to_i) 
+          end
+          return literal
+
+        when :attribute
+          new_nodeset = []
+          case path_stack.shift
+          when :qname
+            prefix = path_stack.shift
+            name = path_stack.shift
+            for element in nodeset
+              if element.node_type == :element
+                for attribute_name in element.attributes.keys
+                  if attribute_name.downcase == name.downcase
+                    attrib = element.attribute( attribute_name,
+                      @namespaces[prefix] )
+                    new_nodeset << attrib if attrib
+                  end
+                end
+              end
+            end
+          when :any
+            #puts "ANY"
+            for element in nodeset
+              if element.node_type == :element
+                new_nodeset += element.attributes.to_a
+              end
+            end
+          end
+          nodeset = new_nodeset
+
+        when :parent
+          #puts "PARENT 1: nodeset = #{nodeset}"
+          nodeset = nodeset.collect{|n| n.parent}.compact
+          #nodeset = expr(path_stack.dclone, nodeset.collect{|n| n.parent}.compact)
+          #puts "PARENT 2: nodeset = #{nodeset.inspect}"
+          node_types = ELEMENTS
+
+        when :ancestor
+          new_nodeset = []
+          for node in nodeset
+            while node.parent
+              node = node.parent
+              new_nodeset << node unless new_nodeset.include? node
+            end
+          end
+          nodeset = new_nodeset
+          node_types = ELEMENTS
+
+        when :ancestor_or_self
+          new_nodeset = []
+          for node in nodeset
+            if node.node_type == :element
+              new_nodeset << node
+              while ( node.parent )
+                node = node.parent
+                new_nodeset << node unless new_nodeset.include? node
+              end
+            end
+          end
+          nodeset = new_nodeset
+          node_types = ELEMENTS
+
+        when :predicate
+          new_nodeset = []
+          subcontext = { :size => nodeset.size }
+          pred = path_stack.shift
+          nodeset.each_with_index { |node, index|
+            subcontext[ :node ] = node
+            #puts "PREDICATE SETTING CONTEXT INDEX TO #{index+1}"
+            subcontext[ :index ] = index+1
+            pc = pred.dclone
+            #puts "#{node.hash}) Recursing with #{pred.inspect} and [#{node.inspect}]"
+            result = expr( pc, [node], subcontext )
+            result = result[0] if result.kind_of? Array and result.length == 1
+            #puts "#{node.hash}) Result = #{result.inspect} (#{result.class.name})"
+            if result.kind_of? Numeric
+              #puts "Adding node #{node.inspect}" if result == (index+1)
+              new_nodeset << node if result == (index+1)
+            elsif result.instance_of? Array
+              #puts "Adding node #{node.inspect}" if result.size > 0
+              new_nodeset << node if result.size > 0
+            else
+              #puts "Adding node #{node.inspect}" if result
+              new_nodeset << node if result
+            end
+          }
+          #puts "New nodeset = #{new_nodeset.inspect}"
+          #puts "Path_stack  = #{path_stack.inspect}"
+          nodeset = new_nodeset
+=begin
+          predicate = path_stack.shift
+          ns = nodeset.clone
+          result = expr( predicate, ns )
+          #puts "Result = #{result.inspect} (#{result.class.name})"
+          #puts "nodeset = #{nodeset.inspect}"
+          if result.kind_of? Array
+            nodeset = result.zip(ns).collect{|m,n| n if m}.compact
+          else
+            nodeset = result ? nodeset : []
+          end
+          #puts "Outgoing NS = #{nodeset.inspect}"
+=end
+
+        when :descendant_or_self
+          rv = descendant_or_self( path_stack, nodeset )
+          path_stack.clear
+          nodeset = rv
+          node_types = ELEMENTS
+
+        when :descendant
+          results = []
+          nt = nil
+          for node in nodeset
+            nt = node.node_type
+            results += expr( path_stack.dclone.unshift( :descendant_or_self ),
+              node.children ) if nt == :element or nt == :document
+          end
+          nodeset = results
+          node_types = ELEMENTS
+
+        when :following_sibling
+          #puts "FOLLOWING_SIBLING 1: nodeset = #{nodeset}"
+          results = []
+          for node in nodeset
+            all_siblings = node.parent.children
+            current_index = all_siblings.index( node )
+            following_siblings = all_siblings[ current_index+1 .. -1 ]
+            results += expr( path_stack.dclone, following_siblings )
+          end
+          #puts "FOLLOWING_SIBLING 2: nodeset = #{nodeset}"
+          nodeset = results
+
+        when :preceding_sibling
+          results = []
+          for node in nodeset
+            all_siblings = node.parent.children
+            current_index = all_siblings.index( node )
+            preceding_siblings = all_siblings[ 0 .. current_index-1 ].reverse
+            #results += expr( path_stack.dclone, preceding_siblings )
+          end
+          nodeset = preceding_siblings
+          node_types = ELEMENTS
+
+        when :preceding
+          new_nodeset = []
+          for node in nodeset
+            new_nodeset += preceding( node )
+          end
+          #puts "NEW NODESET => #{new_nodeset.inspect}"
+          nodeset = new_nodeset
+          node_types = ELEMENTS
+
+        when :following
+          new_nodeset = []
+          for node in nodeset
+            new_nodeset += following( node )
+          end
+          nodeset = new_nodeset
+          node_types = ELEMENTS
+
+        when :namespace
+          new_set = []
+          for node in nodeset
+            new_nodeset << node.namespace if node.node_type == :element or node.node_type == :attribute
+          end
+          nodeset = new_nodeset
+
+        when :variable
+          var_name = path_stack.shift
+          return @variables[ var_name ]
+
+        # :and, :or, :eq, :neq, :lt, :lteq, :gt, :gteq
+        when :eq, :neq, :lt, :lteq, :gt, :gteq, :and, :or
+          left = expr( path_stack.shift, nodeset, context )
+          #puts "LEFT => #{left.inspect} (#{left.class.name})"
+          right = expr( path_stack.shift, nodeset, context )
+          #puts "RIGHT => #{right.inspect} (#{right.class.name})"
+          res = equality_relational_compare( left, op, right )
+          #puts "RES => #{res.inspect}"
+          return res
+
+        when :div
+          left = Functions::number(expr(path_stack.shift, nodeset, context)).to_f
+          right = Functions::number(expr(path_stack.shift, nodeset, context)).to_f
+          return (left / right)
+
+        when :mod
+          left = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          right = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          return (left % right)
+
+        when :mult
+          left = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          right = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          return (left * right)
+
+        when :plus
+          left = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          right = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          return (left + right)
+
+        when :minus
+          left = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          right = Functions::number(expr(path_stack.shift, nodeset, context )).to_f
+          return (left - right)
+
+        when :union
+          left = expr( path_stack.shift, nodeset, context )
+          right = expr( path_stack.shift, nodeset, context )
+          return (left | right)
+
+        when :neg
+          res = expr( path_stack, nodeset, context )
+          return -(res.to_f)
+
+        when :not
+        when :function
+          func_name = path_stack.shift.tr('-','_')
+          arguments = path_stack.shift
+          #puts "FUNCTION 0: #{func_name}(#{arguments.collect{|a|a.inspect}.join(', ')})" 
+          subcontext = context ? nil : { :size => nodeset.size }
+
+          res = []
+          cont = context
+          nodeset.each_with_index { |n, i| 
+            if subcontext
+              subcontext[:node]  = n
+              subcontext[:index] = i
+              cont = subcontext
+            end
+            arg_clone = arguments.dclone
+            args = arg_clone.collect { |arg| 
+              #puts "FUNCTION 1: Calling expr( #{arg.inspect}, [#{n.inspect}] )"
+              expr( arg, [n], cont ) 
+            }
+            #puts "FUNCTION 2: #{func_name}(#{args.collect{|a|a.inspect}.join(', ')})" 
+            Functions.context = cont
+            res << Functions.send( func_name, *args )
+            #puts "FUNCTION 3: #{res[-1].inspect}"
+          }
+          return res
+
+        end
+      end # while
+      #puts "EXPR returning #{nodeset.inspect}"
+      return nodeset
+    end
+  
+    # Monkey Patch for Ruby 1.8.2
+    def internal_parse(path_stack, nodeset) # :nodoc:
+      return nodeset if nodeset.size == 0 or path_stack.size == 0
+      case path_stack.shift
+      when :document
+        return [ nodeset[0].root.parent ]
+
+      when :qname
+        prefix = path_stack.shift.downcase
+        name = path_stack.shift.downcase
+        n = nodeset.clone
+        ns = @namespaces[prefix]
+        ns = ns ? ns : ''
+        n.delete_if do |node|
+          if node.node_type == :element and ns == ''
+            ns = node.namespace( prefix )
+          end
+          !(node.node_type == :element and
+            node.name.downcase == name.downcase and node.namespace == ns )
+        end
+        return n
+
+      when :any
+        n = nodeset.clone
+        n.delete_if { |node| node.node_type != :element }
+        return n
+
+      when :self
+        # THIS SPACE LEFT INTENTIONALLY BLANK
+
+      when :processing_instruction
+        target = path_stack.shift
+        n = nodeset.clone
+        n.delete_if do |node|
+          (node.node_type != :processing_instruction) or 
+          ( !target.nil? and ( node.target != target ) )
+        end
+        return n
+
+      when :text
+        n = nodeset.clone
+        n.delete_if do |node|
+          node.node_type != :text
+        end
+        return n
+
+      when :comment
+        n = nodeset.clone
+        n.delete_if do |node|
+          node.node_type != :comment
+        end
+        return n
+
+      when :node
+        return nodeset
+      
+      when :child
+        new_nodeset = []
+        nt = nil
+        for node in nodeset
+          nt = node.node_type
+          new_nodeset += node.children if nt == :element or nt == :document
+        end
+        return new_nodeset
+
+      when :literal
+        literal = path_stack.shift
+        if literal =~ /^\d+(\.\d+)?$/
+          return ($1 ? literal.to_f : literal.to_i) 
+        end
+        return literal
+        
+      when :attribute
+        new_nodeset = []
+        case path_stack.shift
+        when :qname
+          prefix = path_stack.shift
+          name = path_stack.shift
+          for element in nodeset
+            if element.node_type == :element
+              for attribute_name in element.attributes.keys
+                if attribute_name.downcase == name.downcase
+                  attrib = element.attribute( attribute_name,
+                    @namespaces[prefix] )
+                  new_nodeset << attrib if attrib
+                end
+              end
+            end
+          end
+        when :any
+          for element in nodeset
+            if element.node_type == :element
+              new_nodeset += element.attributes.to_a
+            end
+          end
+        end
+        return new_nodeset
+
+      when :parent
+        return internal_parse( path_stack,
+          nodeset.collect{|n| n.parent}.compact )
+
+      when :ancestor
+        new_nodeset = []
+        for node in nodeset
+          while node.parent
+            node = node.parent
+            new_nodeset << node unless new_nodeset.include? node
+          end
+        end
+        return new_nodeset
+
+      when :ancestor_or_self
+        new_nodeset = []
+        for node in nodeset
+          if node.node_type == :element
+            new_nodeset << node
+            while ( node.parent )
+              node = node.parent
+              new_nodeset << node unless new_nodeset.include? node
+            end
+          end
+        end
+        return new_nodeset
+
+      when :predicate
+        predicate = path_stack.shift
+        new_nodeset = []
+        Functions::size = nodeset.size
+        nodeset.size.times do |index|
+          node = nodeset[index]
+          Functions::node = node
+          Functions::index = index+1
+          result = Predicate( predicate, node )
+          if result.kind_of? Numeric
+            new_nodeset << node if result == (index+1)
+          elsif result.instance_of? Array
+            new_nodeset << node if result.size > 0
+          else
+            new_nodeset << node if result
+          end
+        end
+        return new_nodeset
+
+      when :descendant_or_self
+        rv = descendant_or_self( path_stack, nodeset )
+        path_stack.clear
+        return rv
+
+      when :descendant
+        results = []
+        nt = nil
+        for node in nodeset
+          nt = node.node_type
+          if nt == :element or nt == :document
+            results += internal_parse(
+              path_stack.clone.unshift( :descendant_or_self ),
+              node.children )
+          end
+        end
+        return results
+
+      when :following_sibling
+        results = []
+        for node in nodeset
+          all_siblings = node.parent.children
+          current_index = all_siblings.index( node )
+          following_siblings = all_siblings[ current_index+1 .. -1 ]
+          results += internal_parse( path_stack.clone, following_siblings )
+        end
+        return results
+
+      when :preceding_sibling
+        results = []
+        for node in nodeset
+          all_siblings = node.parent.children
+          current_index = all_siblings.index( node )
+          preceding_siblings = all_siblings[ 0 .. current_index-1 ]
+          results += internal_parse( path_stack.clone, preceding_siblings )
+        end
+        return results
+
+      when :preceding
+        new_nodeset = []
+        for node in nodeset
+          new_nodeset += preceding( node )
+        end
+        return new_nodeset
+
+      when :following
+        new_nodeset = []
+        for node in nodeset
+          new_nodeset += following( node )
+        end
+        return new_nodeset
+
+      when :namespace
+        new_set = []
+        for node in nodeset
+          if node.node_type == :element or node.node_type == :attribute
+            new_nodeset << node.namespace
+          end
+        end
+        return new_nodeset
+
+      when :variable
+        var_name = path_stack.shift
+        return @variables[ var_name ]
+
+      end
+      nodeset
+    end
+  end
+  
+  class XPath # :nodoc:
+    def self.liberal_match(element, path=nil, namespaces={},
+        variables={}) # :nodoc:
+			parser = LiberalXPathParser.new
+			parser.namespaces = namespaces
+			parser.variables = variables
+			path = "*" unless path
+			element = [element] unless element.kind_of? Array
+			parser.parse(path, element)
+    end
+
+    def self.liberal_first(element, path=nil, namespaces={},
+        variables={}) # :nodoc:
+      parser = LiberalXPathParser.new
+      parser.namespaces = namespaces
+      parser.variables = variables
+      path = "*" unless path
+      element = [element] unless element.kind_of? Array
+      parser.parse(path, element)[0]
+    end
+
+    def self.liberal_each(element, path=nil, namespaces={},
+        variables={}, &block) # :nodoc:
+			parser = LiberalXPathParser.new
+			parser.namespaces = namespaces
+			parser.variables = variables
+			path = "*" unless path
+			element = [element] unless element.kind_of? Array
+			parser.parse(path, element).each( &block )
+    end
+  end
+  
+  class Element # :nodoc:
+    unless REXML::Element.public_instance_methods.include? :inner_xml
+      def inner_xml # :nodoc:
+        result = ""
+        self.each_child do |child|
+          if child.kind_of? REXML::Comment
+            result << "<!--" + child.to_s + "-->"
+          else
+            result << child.to_s
+          end
+        end
+        return result.strip
+      end
+    else
+      warn("inner_xml method already exists.")
+    end
+    
+    def base_uri # :nodoc:
+      begin
+        base_attribute = FeedTools::XmlHelper.try_xpaths(self, [
+          '@xml:base'
+        ])
+        if parent == nil || parent.kind_of?(REXML::Document)
+          return nil if base_attribute == nil
+          return base_attribute.value
+        end
+        if base_attribute != nil && parent == nil
+          return base_attribute.value
+        elsif parent != nil && base_attribute == nil
+          return parent.base_uri
+        elsif parent != nil && base_attribute != nil
+          parent_base_uri = parent.base_uri
+          if parent_base_uri != nil
+            uri = URI.parse(parent_base_uri)
+            return (uri + base_attribute.value).to_s
+          else
+            return base_attribute.value
+          end
+        end
+        return nil
+      rescue
+        return nil
+      end
+    end
+  end
+end