feed_discover 0.1.0

data/lib/feed_discover.rb

@@ -0,0 +1,175 @@
+#!/usr/bin/ruby
+#Copyright (C) 2007 Ben Coffey (me@inelegant.org) <http://inelegant.org/>
+#
+#This program is free software; you can redistribute it and/or
+#modify it under the terms of the GNU General Public License
+#as published by the Free Software Foundation; either version 2
+#of the License, or (at your option) any later version.
+#
+#This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#GNU General Public License for more details.
+#
+#You should have received a copy of the GNU General Public License
+#along with this program; if not, write to the Free Software
+#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# FeedDiscover performs feed autodiscovery on the given URL. 
+#
+# Feed autodiscovery is a process of using standardised heruitsics
+# to determine the Atom/RSS feed(s) associated with a given X?HTML
+# document. The process is being standardised for Atom feeds via the 
+# IETF: http://tools.ietf.org/html/draft-ietf-atompub-autodiscovery 
+# We use the same approach for RSS feeds.
+#
+# When feeds cannot be discovered using the above method, we use a
+# broader, thus less accurate, method of looking for hyperlinks whose
+# URLs end with an extension typical of feeds. This feature is experimental.
+#
+# Author:: Ben Coffey  (mailto:me@inelegant.org)
+# Copyright:: Copyright (c) 2007 Ben Coffey
+# License:: GPL
+
+require 'rubygems'
+require 'uri'
+require 'simple_http'
+
+class Array #:nodoc: all
+  def to_h(default=nil) 
+    Hash[ *inject([]) { |a, value| a.push value, default || yield(value) } ] 
+  end
+end
+
+#This class performs feed autodiscovery. This is is the process of 
+#determining the Atom/RSS URL(s) for a given X?HTML page.
+#
+#== Sample Usage
+#
+# require 'rubygems'
+# require 'feed_discover'
+# fd = FeedDiscover.new('http://bbc.co.uk/')
+# if fd.feeds?
+#   puts fd.feeds.join("\n")
+# else
+#   puts "No feeds found"
+# end
+#
+#Or:
+#
+# require 'rubygems'
+# require 'feed_discover'
+# puts FeedDiscover.new('http://inelegant.org/').feed
+#
+class FeedDiscover
+  #The class should be initialized with the URL of the page to perform 
+  #autodiscovery on. The URL should be absolute and use either the HTTP
+  #or HTTP protocol scheme. The URL should be a string or an object whose
+  #+to_s+ method returns a string.
+  #An exception will be raised if the URL has invalid syntax.
+  def initialize(url)
+    @url = URI.parse(url.to_s)
+    @feeds = []
+    autodiscover
+  end
+  
+  #Returns an array of absolute feed URLs which were discovered. If no feeds
+  #were found, an empty array is returned.
+  def feeds
+    @feeds || []
+  end
+  
+  #Returns the absolute URL of the first feed found as a string
+  def feed
+    @feeds[0]
+  end
+  
+  #Returns true if feeds were found; false if not
+  def feeds?
+    !feeds.empty?
+  end
+
+  private
+  #TODO: Add to URI class
+  #The feed URLs we return need to be absolute. If they're already
+  #absolute, we just return them as-is. If they're not, we attempt
+  #to absolutize them relative to the current base URL
+  def abs_uri_wrt_base(str)
+    #In case we're passed a URI object, we normalise it to a string.
+    #We remove leading/trailing whitespace because the values of
+    #'href' attributes sometimes contain such characters
+    str = str.to_s.strip
+    return URI.parse(str) if URI.parse(str).absolute?
+    return @url.merge(str) if @url.absolute?
+    raise "abs_uri_wrt_base: neither URL absolute!"  
+  end
+
+  #Valid MIME types for feeds
+  VALID_MIME_TYPE = %w{application/x.atom+xml application/atom+xml application/xml 
+                       text/xml application/rss+xml application/rdf+xml}.to_h(true)
+  #Valid extensions for feeds
+  VALID_EXT = /\.(?:rss|xml|rdf)$/
+
+  def autodiscover
+    res = SimpleHttp.new @url
+    #TODO: How to handle exceptions?
+    @body = res.get || return
+    #If the URL that we were called with is already a feed (IOW,
+    #its MIME type matches one of the keys of VALID_MIME_TYPE), 
+    #just return it. Autodiscovery is only specified for X?HTML
+    #documents.
+    if VALID_MIME_TYPE[res.response_headers['content-type']]
+      @feeds << @url.to_s
+      return
+    end
+    find_links
+  end
+
+  def find_links
+    require 'rubyful_soup'
+    require 'htmlentities'
+    coder = HTMLEntities.new
+    soup = BeautifulSoup.new(@body)
+    soup.find_all(['a', 'base', 'link']).each do |tag|
+      #link tags must have 'rel', 'type', and 'href' attributes to be considered
+      if tag.name == 'link' and tag['rel'] and tag['type'] and tag['href']
+        #The 'type' attribute must contain a valid MIME type, as defined in
+        #VALID_MIME_TYPE. We ignore its case and any trailing whitespace.
+        #X?HTML entities are decoded performing the lookup.
+        next unless VALID_MIME_TYPE[coder.decode(tag['type']).downcase.strip]
+        #The 'rel' attribute contains a whitespace-separated list of values,
+        #which we must consider regardless of case. Any X?HTML entities are
+        #decoded. We're interested in the presence of one of two specific
+        #values, so we map the value list to a hash-based lookup table
+        rel = coder.decode(tag['rel']).downcase.split(/\s+/).to_h(true)
+        #We use the 'rel' lookup table we created above to determine whether
+        #this link tag is a valid source for feed links. If it is, we
+        #resolve the URL (the value of its 'href' attribute) relative to
+        #the current base URL, then add the result to the @feeds array
+        @feeds << abs_uri_wrt_base(tag['href']).to_s if (rel['alternate'] || rel['service.feed'])
+      elsif tag.name == 'base' and tag['href']
+        #By default, the base URL is equal to the URL we were passed in the
+        #constructor; the URL of the HTML page. However, if the document
+        #contains a base tag with an 'href' attribute, we use that instead.
+        #All discovered feed URLs are resolved relative to the base URL.
+        #TODO: Investigate repercussions of multiple base URLs -- which
+        #one should we use?
+        @url = abs_uri_wrt_base(tag['href'])
+      elsif @feeds.empty? and tag.name == 'a' and tag['href']
+        #If we didn't find any feeds via the 'link' tags, we switch to another
+        #heuristic: look for hyperlinks ('a' tags) whose 'href' attributes
+        #contain a URL ending with an extension typical for a feed (VALID_EXT).
+        #TODO: Limit the number of URLs found by this method such that blogs
+        #with blogroll listings don't generate false positives.
+        begin
+          href = abs_uri_wrt_base(tag['href'])
+        rescue URI::InvalidURIError
+          next
+        end
+        @feeds << href.to_s if VALID_EXT.match(href.path) 
+      end
+    end
+  end
+  
+end
+

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: feed_discover
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2007-05-18 00:00:00 +01:00
+summary: A library for performing feed autodiscovery on X?HTML pages
+require_paths: 
+- lib
+email: me@inelegant.org
+homepage: http://inelegant.org/feed-discover/
+rubyforge_project: 
+description: This library uses heuristics to determine the Atom/RSS feed associated with a given web page.
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: false
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Ben Coffey
+files: 
+- lib/feed_discover.rb
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rubyful_soup
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.4
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: simplehttp
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.1.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: htmlentities
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 4.0.0
+    version: