feedparser 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6333596837d069834ffd82fe303bc4d11eea956
-  data.tar.gz: 8d53dd07e144f4d0c87a1c02373df25826514579
+  metadata.gz: 520ebca7dbff20c3347536869aa6dcfd87d68ddc
+  data.tar.gz: 5378e9fc3d8038ed61d085999fcb7d1ab3e1f041
 SHA512:
-  metadata.gz: 119e6107807ae4c7781379d2e04e255d9906fafef5bbf44a57557a4bfc02fb64f647bce77e9cbad3e7546da5a852775db96ee11e4fb51be7936cf02b03d82463
-  data.tar.gz: fac5a2f23e61173b65b0d52711651b02b4d5d7585a52eed12416acc96256167468fa92b31b837888e1e1d09cec347ce1554be56c9078958711e2f246aaeec57f
+  metadata.gz: 40cc695312a0007da9e569fd05dd35d6621e08367248191eea645c77a93ed5e64c58c0febd9b9d5762177751b3e9c712adbcd52c1ec951e9125c085719f37734
+  data.tar.gz: 96fb8f2c92a83447cc5598b667a3a5036d66c93305b3a624a9f6081e4489b43f5bf37b51cc61959f0ad1a621bd6d1c456a514a1c501e7112179f00d6f1c55cfb

data/Manifest.txt

@@ -3,13 +3,19 @@ Manifest.txt
 README.md
 Rakefile
 lib/feedparser.rb
+lib/feedparser/author.rb
 lib/feedparser/builder/atom.rb
 lib/feedparser/builder/json.rb
 lib/feedparser/builder/rss.rb
 lib/feedparser/feed.rb
+lib/feedparser/generator.rb
 lib/feedparser/item.rb
 lib/feedparser/parser.rb
+lib/feedparser/tag.rb
 lib/feedparser/version.rb
+test/feeds/books/nostarch.rss
+test/feeds/books/oreilly.feedburner.atom
+test/feeds/books/pragprog.rss
 test/feeds/byparker.json
 test/feeds/daringfireball.atom
 test/feeds/daringfireball.json
@@ -19,22 +25,45 @@ test/feeds/headius.atom
 test/feeds/inessential.json
 test/feeds/intertwingly.atom
 test/feeds/jsonfeed.json
-test/feeds/lambdatheultimate.rss2
-test/feeds/learnenough.atom
-test/feeds/railstutorial.atom
-test/feeds/rubyflow.rss2
-test/feeds/rubymine.rss2
+test/feeds/lambdatheultimate.rss
+test/feeds/learnenough.feedburner.atom
+test/feeds/news/nytimes-blogs-bits.rss
+test/feeds/news/nytimes-paul-krugman.rss
+test/feeds/news/nytimes-tech.rss
+test/feeds/news/nytimes-thomas-l-friedman.rss
+test/feeds/news/nytimes.rss
+test/feeds/news/washingtonpost-blogs-innovations.rss
+test/feeds/news/washingtonpost-politics.rss
+test/feeds/news/washingtonpost-world.rss
+test/feeds/ongoing.atom
+test/feeds/osm/blog.openstreetmap.rss
+test/feeds/osm/blogs.openstreetmap.rss
+test/feeds/osm/mapbox.rss
+test/feeds/railstutorial.feedburner.atom
+test/feeds/rubyflow.feedburner.rss
+test/feeds/rubymine.feedburner.rss
 test/feeds/rubyonrails.atom
-test/feeds/sitepoint.rss2
+test/feeds/scripting.rss
+test/feeds/sitepoint.rss
+test/feeds/spec/atom/author.atom
+test/feeds/spec/atom/authors.atom
+test/feeds/spec/atom/categories.atom
 test/feeds/spec/json/example.json
 test/feeds/spec/json/microblog.json
+test/feeds/spec/json/tags.json
+test/feeds/spec/rss/author.rss
+test/feeds/spec/rss/categories.rss
+test/feeds/spec/rss/creator.rss
 test/feeds/xkcd.atom
-test/feeds/xkcd.rss2
+test/feeds/xkcd.rss
 test/helper.rb
 test/test_atom.rb
 test/test_atom_live.rb
+test/test_authors.rb
+test/test_books.rb
 test/test_dates.rb
 test/test_feeds.rb
 test/test_json.rb
 test/test_rss.rb
 test/test_rss_live.rb
+test/test_tags.rb

data/README.md

@@ -11,14 +11,21 @@ feedparser gem - web feed parser and normalizer (Atom, RSS 2.0, JSON, etc.)
 
 ## What's News?
 
-May/2017: Added support for reading feeds in the new [JSON Feed](https://jsonfeed.org) format in - surprise, surprise - JSON. 
+May/2017: Added support for reading feeds in the new [JSON Feed](https://jsonfeed.org) format in - surprise, surprise - JSON.
+
+## What's a Web Feed?
+
+See the [Awesome Feeds](https://github.com/statictimes/awesome-feeds) page ».
+
 
 
 ## Usage
 
+
 ### Structs
 
-Feed • Item
+Feed • Item • Author • Tag • Generator
+
 
 ![](feed-models.png)
 
@@ -30,7 +37,7 @@ Note: uses question mark (`?`) for optional elements (otherwise assume required
 
 **Title 'n' Summary**
 
-Note: The Feed parser will remove all html tags and attributes from the title (RSS 2.0+Atom), 
+Note: The Feed parser will remove all html tags and attributes from the title (RSS 2.0+Atom),
 description (RSS 2.0) and subtitle (Atom) content and will unescape HTML entities e.g. `&` becomes & and so on - always
 resulting in plain vanilla text.
 
@@ -47,7 +54,7 @@ resulting in plain vanilla text.
 | `feed.updated`     | `lastBuildDate`?    | RFC-822 format    | `updated`  | ISO 801 format  |
 | `feed.published`   | `pubDate`?          | RFC-822 format    |  -         |                 |
 
-Note: Check - for RSS 2.0 set feed.updated to pubDate or lastBuildDate if only one present? if both present - map as above. 
+Note: Check - for RSS 2.0 set feed.updated to pubDate or lastBuildDate if only one present? if both present - map as above.
 
 
 RFC-822 date format e.g. Wed, 14 Jan 2015 19:48:57 +0100
@@ -74,12 +81,14 @@ class Feed
 end
 ```
 
+(Source: [`lib/feedparser/feed.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/feed.rb))
+
 
 ### `Item` Struct
 
 **Title 'n' Summary**
 
-Note: The Feed parser will remove all html tags and attributes from the title (RSS 2.0+Atom), 
+Note: The Feed parser will remove all html tags and attributes from the title (RSS 2.0+Atom),
 description (RSS 2.0) and summary (Atom) content
 and will unescape HTML entities e.g. `&` becomes & and so on - always
 resulting in plain vanilla text.
@@ -98,7 +107,7 @@ Note: The content element will assume html content.
 
 | Feed Struct        | RSS 2.0           | Notes       | Atom          | Notes       | JSON               | Notes       |
 | ------------------ | ----------------- | ----------- | ------------- | ----------- | -----------------  | ----------- |  
-| `item.title`       | `title`           | plain text  | `title`       | plain text  |  `title`?          | plain text  | 
+| `item.title`       | `title`           | plain text  | `title`       | plain text  |  `title`?          | plain text  |
 | `item.summary`     | `description`     | plain text  | `summary`?    | plain text  |   -tbd-            |             |
 | `item.content`     | `content`?        | html        | `content`?    | html        |  `content_html (*)`| html        |
 
@@ -129,6 +138,23 @@ class Item
 end
 ```
 
+(Source: [`lib/feedparser/item.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/item.rb))
+
+
+### `Author` Struct
+
+(Source: [`lib/feedparser/author.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/author.rb))
+
+
+### `Tag` Struct
+
+(Source: [`lib/feedparser/tag.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/tag.rb))
+
+### `Generator` Struct
+
+(Source: [`lib/feedparser/generator.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/generator.rb))
+
+
 
 ### Read Feed Example
 
@@ -136,22 +162,131 @@ end
 require 'open-uri'
 require 'feedparser'
 
-txt = open( 'http://openfootball.github.io/atom.xml' ).read
+txt = open( 'http://openfootball.github.io/feed.xml' ).read
 
 feed = FeedParser::Parser.parse( txt )
-pp feed
+
+puts feed.title
+# => "football.db - Open Football Data"
+
+puts feed.url
+# => "http://openfootball.github.io/"
+
+puts feed.items[0].title
+# => "football.db - League Quick Starter Sample - Mauritius Premier League - Create Your Own Repo/League(s) from Scratch"
+
+puts feed.items[0].url
+# => "http://openfootball.github.io/2015/08/30/league-quick-starter.html"
+
+puts feed.items[0].updated
+# => Sun, 30 Aug 2015 00:00:00 +0000
+
+puts feed.items[0].content
+# => "Added a new quick starter sample using the Mauritius Premier League to get you started..."
+
+...
 ```
 
 or reading a feed in the new [JSON Feed](https://jsonfeed.org) format in - surprise, surprise - JSON;
 note: nothing changes :-)
 
 ```
-txt = open( 'https://jsonfeed.org/feed.json' ).read
+txt = open( 'http://openfootball.github.io/feed.json' ).read
 
 feed = FeedParser::Parser.parse( txt )
-pp feed
+
+puts feed.title
+# => "football.db - Open Football Data"
+
+puts feed.url
+# => "http://openfootball.github.io/"
+
+puts feed.items[0].title
+# => "football.db - League Quick Starter Sample - Mauritius Premier League - Create Your Own Repo/League(s) from Scratch"
+
+puts feed.items[0].url
+# => "http://openfootball.github.io/2015/08/30/league-quick-starter.html"
+
+puts feed.items[0].updated
+# => Sun, 30 Aug 2015 00:00:00 +0000
+
+puts feed.items[0].content_text
+# => "Added a new quick starter sample using the Mauritius Premier League to get you started..."
+
+...
 ```
 
+## Samples
+
+### Feed Reader
+
+_Planet Feed Reader in 20 Lines of Ruby_
+
+`planet.rb`:
+
+``` ruby
+require 'open-uri'
+require 'feedparser'
+require 'erb'
+
+# step 1) read a list of web feeds
+
+FEED_URLS = [
+  'http://vienna-rb.at/atom.xml',
+  'http://weblog.rubyonrails.org/feed/atom.xml',
+  'http://www.ruby-lang.org/en/feeds/news.rss',
+  'http://openfootball.github.io/feed.json',
+]
+
+items = []
+
+FEED_URLS.each do |url|
+  feed = FeedParser::Parser.parse( open( url ).read )
+  items += feed.items
+end
+
+# step 2) mix up all postings in a new page
+
+FEED_ITEM_TEMPLATE = <<EOS
+<% items.each do |item| %>
+  <div class="item">
+    <h2><a href="<%= item.url %>"><%= item.title %></a></h2>
+    <div><%= item.content %></div>
+  </div>
+<% end %>
+EOS
+
+puts ERB.new( FEED_ITEM_TEMPLATE ).result
+```
+
+Run the script:
+
+```
+$ ruby ./planet.rb      
+```
+
+Prints:
+
+```
+<div class="item">
+  <h2><a href="http://vienna-rb.at/blog/2017/11/06/picks/">Picks / what the vienna.rb team thinks is worth sharing this week</a></h2>
+  <div>
+   <h3>6/11 Picks!!</h3>
+   <p>In a series on this website we'll entertain YOU with our picks...
+ ...
+```
+
+## Real World Usage
+
+See the Planet Pluto feed reader family:
+
+- [Planet Pluto](https://github.com/feedreader)  - static planet website generator
+- [Planet Pluto Live](https://github.com/plutolive) - dynamic (live) planet web apps (using Sinatra, Rails, etc.)
+
+
+Add your tools, scripts, apps here! Let us know.
+
+
 
 ## Install
 

data/Rakefile

@@ -8,7 +8,7 @@ Hoe.spec 'feedparser' do
   self.summary = 'feedparser - web feed parser and normalizer (RSS 2.0, Atom, JSON, etc.)'
   self.description = summary
 
-  self.urls    = ['https://github.com/feedreader/feedparser']
+  self.urls    = ['https://github.com/feedparser/feedparser']
 
   self.author  = 'Gerald Bauer'
   self.email   = 'wwwmake@googlegroups.com'

data/lib/feedparser.rb

@@ -26,6 +26,9 @@ require 'feedparser/builder/json'
 
 require 'feedparser/feed'
 require 'feedparser/item'
+require 'feedparser/author'
+require 'feedparser/tag'
+require 'feedparser/generator'
 require 'feedparser/parser'
 
 

data/lib/feedparser/author.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+module FeedParser
+
+class Author
+
+  attr_accessor :name
+  attr_accessor :url
+  ## note: uri is an alias for url
+  alias :uri  :url       ## add atom alias for uri - why? why not?
+  alias :uri= :url=
+
+  def email?()   @email.nil? == false;  end
+  attr_accessor :email
+
+  def avatar?()  @avatar.nil? == false;  end
+  attr_accessor :avatar  # todo/check: use avatar_url ?? used by json feed -check if always a url
+
+
+  ## todo: add role - why? why not?
+  ##   e.g. add contributor (atom)
+  ##          or managingEditor (rss) or webMaster (rss) - why? why not??
+
+  attr_accessor :text    # note: holds "unparsed" text (content) line form dc:creator or rss:author
+
+
+  def to_s
+    ## note: to_s  - allows to use just author in templates
+    ##    will by default return name if present or as fallback "unparsed" text line
+     if @name    ## not blank
+       @name
+     else
+       @text
+     end
+  end
+
+end  # class Author
+
+end # module FeedParser

data/lib/feedparser/builder/atom.rb

@@ -22,7 +22,7 @@ class AtomFeedBuilder
 
 
 
-  def build_feed( atom_feed )
+  def build_feed( atom_feed )    ## fix/todo: rename atom_feed to atom or wire or xml or in ???
     feed = Feed.new
     feed.format = 'atom'
 
@@ -36,9 +36,8 @@ class AtomFeedBuilder
     atom_feed.links.each_with_index do |link,i|
       logger.debug "  atom | feed.link[#{i+1}]  rel=>#{link.rel}< : #{link.rel.class.name} type=>#{link.type}< href=>#{link.href}<"
 
-      if link.rel == 'self'
+      if feed.feed_url.nil? && link.rel == 'self'
         feed.feed_url = link.href
-        break
       end
     end
 
@@ -74,18 +73,19 @@ class AtomFeedBuilder
     end
 
 
-    if atom_feed.updated
-      feed.updated = handle_date( atom_feed.updated, 'feed.updated' )
+    if atom_feed.updated && atom_feed.updated.content    ## note: content might be nil if <updated></updated> empty
+      feed.updated_local = handle_date( atom_feed.updated, 'feed.updated' )
+      feed.updated       = feed.updated_local.utc
     end
 
     if atom_feed.generator
       ## Note: remove (strip) leading and trailing spaces and newlines
-      feed.generator =  atom_feed.generator.content.strip
+      feed.generator.name =  atom_feed.generator.content.strip
       logger.debug "  atom | feed.generator.content  >#{atom_feed.generator.content}< : #{atom_feed.generator.content.class.name}"
 
       # pp atom_feed.generator
-      feed.generator_version = atom_feed.generator.version
-      feed.generator_uri     = atom_feed.generator.uri
+      feed.generator.version = atom_feed.generator.version
+      feed.generator.url     = atom_feed.generator.uri
       logger.debug "  atom | feed.generator.version  >#{atom_feed.generator.version}< : #{atom_feed.generator.version.class.name}"
       logger.debug "  atom | feed.generator.uri      >#{atom_feed.generator.uri}< : #{atom_feed.generator.uri.class.name}"
     end
@@ -95,17 +95,52 @@ class AtomFeedBuilder
     end
 
 
-    items = []
+    ## check for authors
+    atom_feed.authors.each do |atom_author|
+      feed.authors << build_author( atom_author )
+    end
+
+    ## check for categories/tags
+    atom_feed.categories.each do |atom_cat|
+      feed.tags << build_tag( atom_cat )
+    end
+
+
     atom_feed.items.each do |atom_item|
-      items << build_feed_item( atom_item )
+      feed.items << build_item( atom_item )
     end
-    feed.items = items
 
     feed # return new feed
   end # method build_feed_from_atom
 
 
-  def build_feed_item( atom_item )
+  def build_author( atom_author )
+    ## pp atom_author
+    author = Author.new
+
+    ## note: always strip leading n trailing spaces (from content)
+    author.name  = atom_author.name.content.strip    if atom_author.name
+    author.url   = atom_author.uri.content.strip     if atom_author.uri
+    author.email = atom_author.email.content.strip   if atom_author.email
+
+    author
+  end  # build_author
+
+
+  def build_tag( atom_cat )
+    ## pp atom_cat
+    tag = Tag.new
+
+    ## note: always strip leading n trailing spaces
+    ##         and add if preset (not blank/empty e.g. not nil or "")
+    tag.name     = atom_cat.term.strip    if atom_cat.term   && !atom_cat.term.empty?
+    tag.scheme   = atom_cat.scheme.strip  if atom_cat.scheme && !atom_cat.scheme.empty?
+
+    tag
+  end  # build_tag
+
+
+  def build_item( atom_item )
     item = Item.new   # Item.new
 
     item.title     = handle_content( atom_item.title, 'item.title' )
@@ -136,12 +171,14 @@ class AtomFeedBuilder
     end
 
 
-    if atom_item.updated
-      item.updated    = handle_date( atom_item.updated, 'item.updated' )
+    if atom_item.updated && atom_item.updated.content
+      item.updated_local  = handle_date( atom_item.updated, 'item.updated' )
+      item.updated        = item.updated_local.utc
     end
 
-    if atom_item.published
-      item.published  = handle_date( atom_item.published, 'item.published' )
+    if atom_item.published && atom_item.published.content 
+      item.published_local  = handle_date( atom_item.published, 'item.published' )
+      item.published        = item.published_local.utc
     end
 
 
@@ -156,8 +193,18 @@ class AtomFeedBuilder
       item.summary = handle_content( atom_item.summary, 'item.summary' )
     end
 
+    ## check for authors
+    atom_item.authors.each do |atom_author|
+      item.authors << build_author( atom_author )
+    end
+
+    ## check for categories/tags
+    atom_item.categories.each do |atom_cat|
+      item.tags << build_tag( atom_cat )
+    end
+
     item
-  end # method build_feed_item
+  end # method build_item