feedparser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ec4aa58ed835c53c649f6289eebf236e25802c69
+  data.tar.gz: e536aa9a2bd29ce9beb2f22765d40c5a027c920f
+SHA512:
+  metadata.gz: 50fe366a7bd6d003f280a0c291d2569ad5122f78fbe7c48f988264ddd32244182ec9684bd8c4e26ea14215c143a7d6770704dee975743b98c7a10fdd34620c1e
+  data.tar.gz: f6d0210aa31b7b9878a25d27c208efe47a0aca840a7253116ea04f3ec4a1da51da069a07d7741f4e645263a440e1b4245a8df677534a70d536d5f3b10348a72c

data/HISTORY.md

@@ -0,0 +1,3 @@
+### 0.1.0 / 2013-09-19
+
+* Everything is new. First release.

data/Manifest.txt

@@ -0,0 +1,20 @@
+HISTORY.md
+Manifest.txt
+README.md
+Rakefile
+lib/feedparser.rb
+lib/feedparser/builder/atom.rb
+lib/feedparser/builder/rss.rb
+lib/feedparser/feed.rb
+lib/feedparser/helper/atom_v03.rb
+lib/feedparser/item.rb
+lib/feedparser/parser.rb
+lib/feedparser/version.rb
+test/feeds/googlegroups.atom
+test/feeds/googlegroups2.atom
+test/feeds/quirksblog.atom.v03
+test/helper.rb
+test/test_atom.rb
+test/test_atom_from_file.rb
+test/test_atom_v03.rb
+test/test_rss.rb

data/README.md

@@ -0,0 +1,95 @@
+# feedparser
+
+feedparser gems - web feed parser and normalizer (RSS 2.0, Atom, etc.)
+
+* home  :: [github.com/rubylibs/feedutils](https://github.com/rubylibs/feedutils)
+* bugs  :: [github.com/rubylibs/feedutils/issues](https://github.com/rubylibs/feedutils/issues)
+* gem   :: [rubygems.org/gems/feedutils](https://rubygems.org/gems/feedutils)
+* rdoc  :: [rubydoc.info/gems/feedutils](http://rubydoc.info/gems/feedutils)
+* forum :: [groups.google.com/group/feedreader](http://groups.google.com/group/feedreader)
+
+
+## Usage
+
+### Structs
+
+Feed • Item
+
+### `Feed` Struct
+
+~~~
+class Feed
+  attr_accessor :format   # e.g. atom|rss 2.0|etc.
+  attr_accessor :title
+  attr_accessor :title_type  # e.g. text|html|html-escaped  (optional) -use - why?? why not??
+  attr_accessor :url
+
+  attr_accessor :items
+
+  attr_accessor :summary        # e.g. description (rss)
+  attr_accessor :summary_type   # e.g. text|html|html-escaped
+
+  attr_accessor :title2         # e.g. subtitle (atom)
+  attr_accessor :title2_type    # e.g. text|html|html-escaped
+
+  attr_accessor :published
+  attr_accessor :updated
+  attr_accessor :built
+
+  attr_accessor :generator
+  attr_accessor :generator_version  # e.g. @version (atom)
+  attr_accessor :generator_uri      # e.g. @uri     (atom) - use alias url/link ???
+end
+~~~
+
+
+### `Item` Struct
+
+~~~
+class Item
+  attr_accessor :title
+  attr_accessor :title_type    # optional for now (text|html|html-escaped) - not yet set
+  attr_accessor :url           # todo: rename to link (use alias) ??
+
+  attr_accessor :content
+  attr_accessor :content_type  # optional for now (text|html|html-escaped|binary-base64) - not yet set
+
+  attr_accessor :summary
+  attr_accessor :summary_type  # optional for now (text|html|html-escaped) - not yet set
+
+  attr_accessor :published
+  attr_accessor :updated
+
+  attr_accessor :guid     # todo: rename to id (use alias) ??
+end
+~~~
+
+
+### Read Feed Example
+
+~~~
+require 'open-uri'
+require 'feedutils'
+
+xml = open( 'http://openfootball.github.io/atom.xml' ).read
+
+feed = FeedUtils::Parser.parse( xml )
+pp feed
+~~~
+
+
+
+## Alternatives
+
+- [`syndication`](http://syndication.rubyforge.org) [(Source)](https://github.com/lpar/syndication) - by Mathew (aka lpar);  RSS 1.0, 2.0, Atom, and understands namespaces; optional support for Dublin Core, iTunes/podcast feeds, and RSS 1.0 Syndication and Content modules
+- [`simple-rss`](http://rubyforge.org/projects/simple-rss)
+- [`feedtools`](http://rubyforge.org/projects/feedtools)
+
+TBD
+
+
+## License
+
+The `feedutils` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+

data/Rakefile

@@ -0,0 +1,32 @@
+require 'hoe'
+require './lib/feedparser/version.rb'
+
+Hoe.spec 'feedparser' do
+
+  self.version = FeedParser::VERSION
+
+  self.summary = 'feedparser - web feed parser and normalizer (RSS 2.0, Atom, etc.)'
+  self.description = summary
+
+  self.urls    = ['https://github.com/feedreader/feed.parser']
+
+  self.author  = 'Gerald Bauer'
+  self.email   = 'feedreader@googlegroups.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'HISTORY.md'
+
+  self.extra_deps = [
+    ['logutils', '>= 0.6.1']
+  ]
+
+  ###  todo: add fetcher dep for testing (e.g. development only)
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+   required_ruby_version: '>= 1.9.2'
+  }
+
+end

data/lib/feedparser.rb

@@ -0,0 +1,27 @@
+# core and stdlibs
+
+require 'rss'
+require 'pp'
+require 'date'
+
+# 3rd party gems/libs
+
+require 'logutils'
+
+# our own code
+
+require 'feedparser/version'  # let it always go first
+
+require 'feedparser/builder/atom'
+require 'feedparser/builder/rss'
+
+require 'feedparser/helper/atom_v03'
+
+require 'feedparser/feed'
+require 'feedparser/item'
+require 'feedparser/parser'
+
+
+
+# say hello
+puts FeedParser.banner     if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG)

data/lib/feedparser/builder/atom.rb

@@ -0,0 +1,141 @@
+
+module FeedParser
+
+class AtomFeedBuilder
+
+  include LogUtils::Logging
+
+  def initialize( atom_feed )
+    @feed = build_feed( atom_feed )
+  end
+
+  def to_feed
+    @feed
+  end
+
+  def self.build( atom_feed )
+    feed = self.new( atom_feed )
+    feed.to_feed
+  end
+
+
+  def build_feed( atom_feed )
+    feed = Feed.new
+    ## feed.object = atom_feed  # not use for now
+    feed.format = 'atom'
+
+    feed.title  = atom_feed.title.content
+    logger.debug "  atom | title.content  >#{atom_feed.title.content}< : #{atom_feed.title.content.class.name}"
+
+
+    logger.debug "  atom | id.content  >#{atom_feed.id.content}< : #{atom_feed.id.content.class.name}"
+
+    feed.url = nil
+
+    ## note: use links (plural to allow multiple links e.g. self,alternate,etc.)
+    atom_feed.links.each_with_index do |link,i|
+      logger.debug "  atom | link[#{i+1}] link rel=>#{link.rel}< : #{link.rel.class.name} type=#{link.type} href=#{link.href}"
+
+      ## for now assume alternate is link or no rel specified (assumes alternate)
+      ##   note: only set if feed.url is NOT already set (via <id> for example)
+      if feed.url.nil? && (link.rel == 'alternate' || link.rel.nil?)
+        feed.url = link.href
+      end
+    end
+
+    ## note: as fallback try id if still no url found
+    ##   use url only if starts_with http
+    ##     might not be link e.g blogger uses for ids =>
+    ##    <id>tag:blogger.com,1999:blog-4704664917418794835</id>
+    ##
+    ##  note: id might actually be link to feed NOT to site  (remove fallback - why - why not???)
+    ##
+    ## Note: remove (strip) leading and trailing spaces and newlines
+
+    if feed.url.nil? && atom_feed.id.content.strip.start_with?( 'http' )
+      feed.url = atom_feed.id.content.strip
+    end
+
+
+    if atom_feed.updated
+      # NOTE: empty updated.content e.g.  used by google groups feed
+      #   will return nil : NilClass
+      
+      ## convert from time to to_datetime  (avoid errors on windows w/ builtin rss lib)
+      
+      feed.updated =  atom_feed.updated.content.nil?  ? nil : atom_feed.updated.content.to_datetime  #  .utc.strftime( "%Y-%m-%d %H:%M" )
+      logger.debug "  atom | updated.content  >#{atom_feed.updated.content}< : #{atom_feed.updated.content.class.name}"
+    end
+
+    if atom_feed.generator
+      ## Note: remove (strip) leading and trailing spaces and newlines
+      feed.generator =  atom_feed.generator.content.strip
+      logger.debug "  atom | generator.content  >#{atom_feed.generator.content}< : #{atom_feed.generator.content.class.name}"
+
+      # pp atom_feed.generator
+      feed.generator_version = atom_feed.generator.version
+      feed.generator_uri     = atom_feed.generator.uri
+      logger.debug "  atom | generator.version  >#{atom_feed.generator.version}< : #{atom_feed.generator.version.class.name}"
+      logger.debug "  atom | generator.uri      >#{atom_feed.generator.uri}< : #{atom_feed.generator.uri.class.name}"
+    end
+
+    if atom_feed.subtitle
+      feed.title2 =  atom_feed.subtitle.content
+      logger.debug "  atom | subtitle.content  >#{atom_feed.subtitle.content}< : #{atom_feed.subtitle.content.class.name}"
+    end
+
+
+    items = []
+    atom_feed.items.each do |atom_item|
+      items << build_feed_item( atom_item )
+    end
+    feed.items = items
+
+    feed # return new feed
+  end # method build_feed_from_atom
+
+  def build_feed_item( atom_item )
+    item = Item.new   # Item.new
+    ## item.object = atom_item  # not used for now
+
+    item.title     = atom_item.title.content
+    item.url       = atom_item.link.href
+
+    logger.debug "  atom | item.title.content: >#{atom_item.title.content}< : #{atom_item.title.content.class.name}"
+    logger.debug "  atom | item.link.href: >#{atom_item.link.href}< : #{atom_item.link.href.class.name}"
+
+
+    if atom_item.updated
+      ## change time to utc if present? why? why not?
+      #  --  .utc.strftime( "%Y-%m-%d %H:%M" )
+
+      ## convert from time to to_datetime  (avoid errors on windows w/ builtin rss lib)
+
+      item.updated    =  atom_item.updated.content.nil? ? nil : atom_item.updated.content.to_datetime
+      logger.debug "  atom | item.updated.content  >#{atom_item.updated.content}< : #{atom_item.updated.content.class.name}"
+    end
+
+    if atom_item.published
+      ## convert from time to to_datetime  (avoid errors on windows w/ builtin rss lib)
+
+      item.published   =  atom_item.published.content.nil? ? nil : atom_item.published.content.to_datetime
+      logger.debug "  atom | item.published.content  >#{atom_item.published.content}< : #{atom_item.published.content.class.name}"
+    end
+
+
+    item.guid       =  atom_item.id.content
+    logger.debug "  atom | item.id.content: >#{atom_item.id.content}< : #{atom_item.id.content.class.name}"
+
+    if atom_item.content
+      item.content = atom_item.content.content
+    end
+
+    if atom_item.summary
+      item.summary = atom_item.summary.content
+    end
+
+    item
+  end # method build_feed_item
+
+end # AtomFeedBuilder
+end # FeedParser

data/lib/feedparser/builder/rss.rb

@@ -0,0 +1,129 @@
+
+module FeedParser
+
+### todo/fix:
+#    rename to Rss20FeedBuilder?? or FeedBuilderRss20 ??
+
+class RssFeedBuilder
+
+  include LogUtils::Logging
+
+  def initialize( rss_feed )
+    @feed = build_feed( rss_feed )
+  end
+
+  def to_feed
+    @feed
+  end
+
+  def self.build( rss_feed )
+    feed = self.new( rss_feed )
+    feed.to_feed
+  end
+
+
+  def build_feed( rss_feed )
+    feed = Feed.new
+    ## feed.object = rss_feed   # not use for now
+    feed.format = "rss #{rss_feed.rss_version}"
+
+    feed.title     = rss_feed.channel.title           # required
+    feed.url       = rss_feed.channel.link            # required
+    feed.summary   = rss_feed.channel.description     # required
+
+    logger.debug "  rss | channel.description: >#{rss_feed.channel.description}< : #{rss_feed.channel.description.class.name}"
+
+     # NOTE:
+     # All date-times in RSS conform
+     #   to the Date and Time Specification of RFC 822
+     #  e.g. Sun, 19 May 2012 15:21:36 GMT  or
+     #       Sat, 07 Sep 2013 00:00:01 GMT
+
+    ## convert from time to to_datetime  (avoid errors on windows w/ builtin rss lib)
+
+    feed.built     = rss_feed.channel.lastBuildDate.nil? ? nil : rss_feed.channel.lastBuildDate.to_datetime  # optional
+    feed.published = rss_feed.channel.pubDate.nil?       ? nil : rss_feed.channel.pubDate.to_datetime     # optional
+
+    logger.debug "  rss | channel.lastBuildDate: >#{rss_feed.channel.lastBuildDate}< : #{rss_feed.channel.lastBuildDate.class.name}"
+    logger.debug "  rss | channel.pubDate: >#{rss_feed.channel.pubDate}< : #{rss_feed.channel.pubDate.class.name}"
+
+
+    feed.generator = rss_feed.channel.generator       # optional
+
+    logger.debug "  rss | channel.generator: >#{rss_feed.channel.generator}< : #{rss_feed.channel.generator.class.name}"
+
+
+    items = []
+    rss_feed.items.each do |rss_item|
+      items << build_feed_item( rss_item )
+    end
+    feed.items = items
+
+    feed # return new feed
+  end
+
+  def build_feed_item( rss_item )
+
+    item = Item.new
+    ## item.object = rss_item  # not use for now
+
+    item.title     = rss_item.title
+    item.url       = rss_item.link
+
+    logger.debug "  rss | item.title: >#{rss_item.title}< : #{rss_item.title.class.name}"
+    logger.debug "  rss | item.link: >#{rss_item.link}< : #{rss_item.link.class.name}"
+
+## todo:
+##  check if feedburner:origLink present - if yes, use it for url/link
+##  example: use
+##  - <feedburner:origLink>http://www.rubyflow.com/items/9803-gotta-ruby-s-syntax</feedburner:origLink></item>
+##   instead of
+##  - <link>http://feedproxy.google.com/~r/Rubyflow/~3/Ym9Sltg_2_c/9803-gotta-ruby-s-syntax</link>
+
+
+    item.summary   = rss_item.description
+
+    # check for <content:encoded>
+    # -- using RSS 1.0 content module in RSS 2.0
+    item.content = rss_item.content_encoded
+    logger.debug "  rss | item.content_encoded[0..40]: >#{rss_item.content_encoded ? rss_item.content_encoded[0..40] : ''}< : #{rss_item.content_encoded.class.name}"
+
+    # NOTE:
+    # All date-times in RSS conform
+    #   to the Date and Time Specification of RFC 822
+    #  e.g. Sun, 19 May 2012 15:21:36 GMT  or
+    #       Sat, 07 Sep 2013 00:00:01 GMT
+
+    ## convert from time to to_datetime  (avoid errors on windows w/ builtin rss lib)
+
+    item.published = rss_item.pubDate.nil? ? nil : rss_item.pubDate.to_datetime    # .utc.strftime( "%Y-%m-%d %H:%M" )
+
+    logger.debug "  rss | item.pubDate: >#{rss_item.pubDate}< : #{rss_item.pubDate.class.name}"
+
+
+    ## fix/todo: check if rss_item.guid present? !!!!
+    ##
+    ##  might be the case e.g. check lambda-the-ultimate.org, for example
+
+    if rss_item.guid && rss_item.guid.content
+      item.guid     = rss_item.guid.content
+      logger.debug "  rss | item.guid.content: >#{rss_item.guid.content}< : #{rss_item.guid.content.class.name}"
+    else
+      item.guid     = rss_item.link
+      logger.warn "  rss | item.guid.content missing !!!! - using link for guid"
+    end
+
+    ### todo: add support or authors (incl. dc:creator)
+    ##  <dc:creator>Dhaivat Pandya</dc:creator>
+
+    #  todo: categories
+    # <category><![CDATA[Gems]]></category>
+    # <category><![CDATA[Ruby]]></category>
+    # <category><![CDATA[Ruby on Rails]]></category>
+
+
+    item
+  end # method build_feed_item_from_rss
+
+end # class RssFeedBuilder
+end # module FeedParser