feedparser 1.2.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 520ebca7dbff20c3347536869aa6dcfd87d68ddc
-  data.tar.gz: 5378e9fc3d8038ed61d085999fcb7d1ab3e1f041
+  metadata.gz: 12a89b6c4d0ad5290a16d44a79245a25bb6ff349
+  data.tar.gz: 63057209038f7cab23eb4fd963d9525b154d2fbb
 SHA512:
-  metadata.gz: 40cc695312a0007da9e569fd05dd35d6621e08367248191eea645c77a93ed5e64c58c0febd9b9d5762177751b3e9c712adbcd52c1ec951e9125c085719f37734
-  data.tar.gz: 96fb8f2c92a83447cc5598b667a3a5036d66c93305b3a624a9f6081e4489b43f5bf37b51cc61959f0ad1a621bd6d1c456a514a1c501e7112179f00d6f1c55cfb
+  metadata.gz: 6220dd036c705fedb52a17495001003c88891a912c3ad9e3fed07045adde3f0902d8dcbf3fc7000fbc297b91bba388979c54fbdaf1890f20e541d9216bfc5438
+  data.tar.gz: 049a3b9cdf4b27fe1fbc2e9109241ed951e4a717e6f620bd6aa7b2b17c9657e1fe468b86fd52cd1bf5ff48192935b3939d6579ca612c5ac576553c2f9bfcb433

data/Manifest.txt

@@ -6,6 +6,7 @@ lib/feedparser.rb
 lib/feedparser/author.rb
 lib/feedparser/builder/atom.rb
 lib/feedparser/builder/json.rb
+lib/feedparser/builder/microformats.rb
 lib/feedparser/builder/rss.rb
 lib/feedparser/feed.rb
 lib/feedparser/generator.rb
@@ -13,57 +14,8 @@ lib/feedparser/item.rb
 lib/feedparser/parser.rb
 lib/feedparser/tag.rb
 lib/feedparser/version.rb
-test/feeds/books/nostarch.rss
-test/feeds/books/oreilly.feedburner.atom
-test/feeds/books/pragprog.rss
-test/feeds/byparker.json
-test/feeds/daringfireball.atom
-test/feeds/daringfireball.json
-test/feeds/googlegroups.atom
-test/feeds/googlegroups2.atom
-test/feeds/headius.atom
-test/feeds/inessential.json
-test/feeds/intertwingly.atom
-test/feeds/jsonfeed.json
-test/feeds/lambdatheultimate.rss
-test/feeds/learnenough.feedburner.atom
-test/feeds/news/nytimes-blogs-bits.rss
-test/feeds/news/nytimes-paul-krugman.rss
-test/feeds/news/nytimes-tech.rss
-test/feeds/news/nytimes-thomas-l-friedman.rss
-test/feeds/news/nytimes.rss
-test/feeds/news/washingtonpost-blogs-innovations.rss
-test/feeds/news/washingtonpost-politics.rss
-test/feeds/news/washingtonpost-world.rss
-test/feeds/ongoing.atom
-test/feeds/osm/blog.openstreetmap.rss
-test/feeds/osm/blogs.openstreetmap.rss
-test/feeds/osm/mapbox.rss
-test/feeds/railstutorial.feedburner.atom
-test/feeds/rubyflow.feedburner.rss
-test/feeds/rubymine.feedburner.rss
-test/feeds/rubyonrails.atom
-test/feeds/scripting.rss
-test/feeds/sitepoint.rss
-test/feeds/spec/atom/author.atom
-test/feeds/spec/atom/authors.atom
-test/feeds/spec/atom/categories.atom
-test/feeds/spec/json/example.json
-test/feeds/spec/json/microblog.json
-test/feeds/spec/json/tags.json
-test/feeds/spec/rss/author.rss
-test/feeds/spec/rss/categories.rss
-test/feeds/spec/rss/creator.rss
-test/feeds/xkcd.atom
-test/feeds/xkcd.rss
 test/helper.rb
-test/test_atom.rb
 test/test_atom_live.rb
-test/test_authors.rb
-test/test_books.rb
 test/test_dates.rb
-test/test_feeds.rb
-test/test_json.rb
-test/test_rss.rb
+test/test_microformats.rb
 test/test_rss_live.rb
-test/test_tags.rb

data/README.md

@@ -1,6 +1,6 @@
 # feedparser
 
-feedparser gem - web feed parser and normalizer (Atom, RSS 2.0, JSON, etc.)
+feedparser gem - web feed parser and normalizer (Atom, RSS 2.0, JSON Feed, HTML h-entry, etc.)
 
 * home  :: [github.com/feedparser/feedparser](https://github.com/feedparser/feedparser)
 * bugs  :: [github.com/feedparser/feedparser/issues](https://github.com/feedparser/feedparser/issues)
@@ -11,11 +11,19 @@ feedparser gem - web feed parser and normalizer (Atom, RSS 2.0, JSON, etc.)
 
 ## What's News?
 
-May/2017: Added support for reading feeds in the new [JSON Feed](https://jsonfeed.org) format in - surprise, surprise - JSON.
+**June/2017**:  Added support for reading feeds in HTML with Microformats incl.
+[`h-entry`](http://microformats.org/wiki/h-entry),
+[`h-feed`](http://microformats.org/wiki/h-feed) and others.
+
+All feed with test assertions for easy (re)use and browsing moved
+to its own repo, that is, [`/feeds`](https://github.com/feedparser/feeds).
+
+**May/2017**: Added support for reading feeds in the new [JSON Feed](https://jsonfeed.org) format in - surprise, surprise - JSON.
+
 
 ## What's a Web Feed?
 
-See the [Awesome Feeds](https://github.com/statictimes/awesome-feeds) page ».
+See the [Awesome Feeds](https://github.com/feedparser/awesome-feeds) page ».
 
 
 
@@ -62,7 +70,7 @@ RFC-822 date format e.g. Wed, 14 Jan 2015 19:48:57 +0100
 ISO-801 date format e.g. 2015-01-11T09:30:16Z
 
 
-```
+``` ruby
 class Feed
   attr_accessor :format   # e.g. atom|rss 2.0|etc.
   attr_accessor :title    # note: always plain vanilla text - if present html tags will get stripped and html entities unescaped
@@ -75,9 +83,9 @@ class Feed
   attr_accessor :updated     # note: is lastBuildDate in RSS 2.0
   attr_accessor :published   # note: is pubDate in RSS 2.0; not available in Atom
 
+  attr_accessor :authors
+  attr_accessor :tags
   attr_accessor :generator
-  attr_accessor :generator_version  # e.g. @version (atom)
-  attr_accessor :generator_uri      # e.g. @uri     (atom) - use alias url/link ???
 end
 ```
 
@@ -121,7 +129,7 @@ Note: The content element will assume html content.
 
 Note: In plain vanilla RSS 2.0 there's only one `pubDate` for items, thus, it's not possible to differeniate between published and updated dates for items; note - the `item.pubDate` will get mapped to `item.updated`. To set the published date in RSS 2.0 use the dublin core module e.g `dc:created`, for example.
 
-```
+``` ruby
 class Item
   attr_accessor :title   # note: always plain vanilla text - if present html tags will get stripped and html entities
   attr_accessor :url
@@ -158,7 +166,7 @@ end
 
 ### Read Feed Example
 
-```
+``` ruby
 require 'open-uri'
 require 'feedparser'
 
@@ -190,7 +198,7 @@ puts feed.items[0].content
 or reading a feed in the new [JSON Feed](https://jsonfeed.org) format in - surprise, surprise - JSON;
 note: nothing changes :-)
 
-```
+``` ruby
 txt = open( 'http://openfootball.github.io/feed.json' ).read
 
 feed = FeedParser::Parser.parse( txt )
@@ -216,6 +224,60 @@ puts feed.items[0].content_text
 ...
 ```
 
+### Microformats
+
+Microformats let you mark up feeds and posts in HTML with
+[`h-entry`](http://microformats.org/wiki/h-entry),
+[`h-feed`](http://microformats.org/wiki/h-feed),
+and friends.
+
+Note: Microformats support in feedparser is optional.
+Install and require the the [microformats gem](https://github.com/indieweb/microformats-ruby) to read
+feeds in HTML with Microformats.
+
+
+``` ruby
+
+require 'microformats'
+
+text =<<HTML
+<article class="h-entry">
+  <h1 class="p-name">Microformats are amazing</h1>
+  <p>Published by
+    <a class="p-author h-card" href="http://example.com">W. Developer</a>
+     on <time class="dt-published" datetime="2013-06-13 12:00:00">13<sup>th</sup> June 2013</time>
+
+  <p class="p-summary">In which I extoll the virtues of using microformats.</p>
+
+  <div class="e-content">
+    <p>Blah blah blah</p>
+  </div>
+</article>
+HTML
+
+feed = FeedParser::Parser.parse( text )
+
+puts feed.format
+# => "html"
+puts feed.items.size
+# =>  1
+puts feed.items[0].authors.size
+# => 1
+puts feed.items[0].content_html  
+# => "<p>Blah blah blah</p>"
+puts feed.items[0].content_text  
+# => "Blah blah blah"
+puts feed.items[0].title
+# => "Microformats are amazing"
+puts feed.items[0].summary
+# => "In which I extoll the virtues of using microformats."
+puts feed.items[0].published
+# => 2013-06-13 12:00:00
+puts feed.items[0].authors[0].name
+# => "W. Developer"
+...
+```
+
 ## Samples
 
 ### Feed Reader

data/Rakefile

@@ -5,7 +5,7 @@ Hoe.spec 'feedparser' do
 
   self.version = FeedParser::VERSION
 
-  self.summary = 'feedparser - web feed parser and normalizer (RSS 2.0, Atom, JSON, etc.)'
+  self.summary = 'feedparser - web feed parser and normalizer (RSS 2.0, Atom, JSON Feed, HTML h-entry, etc.)'
   self.description = summary
 
   self.urls    = ['https://github.com/feedparser/feedparser']

data/lib/feedparser.rb

@@ -23,6 +23,8 @@ require 'feedparser/version'  # let it always go first
 require 'feedparser/builder/atom'
 require 'feedparser/builder/rss'
 require 'feedparser/builder/json'
+require 'feedparser/builder/microformats'
+
 
 require 'feedparser/feed'
 require 'feedparser/item'

data/lib/feedparser/builder/microformats.rb

@@ -0,0 +1,264 @@
+# encoding: utf-8
+
+module FeedParser
+
+
+class HyFeedBuilder
+
+  include LogUtils::Logging
+
+
+  def self.build( hash )
+    feed = self.new( hash )
+    feed.to_feed
+  end
+
+  def initialize( hash )
+    @feed = build_feed( hash )
+  end
+
+  def to_feed
+    @feed
+  end
+
+
+  def build_feed( h )
+
+    b = HyBuilder.new( h )     ## convert hash to structs
+
+    ##  use first feed - more really possible?
+    ##   fix/todo: handle no feed too!!!
+    hy = b.feeds[0]
+
+    ## pp hy
+
+    feed = Feed.new
+    feed.format = 'html'
+
+    ### todo: add
+    ## - feed.title
+    ## - feed.url
+    ## - feed.feed_url
+    ## - feed.summary
+    ## - feed.authors
+    ## etc.
+
+    hy.entries.each do |entry|
+      feed.items << build_item( entry )
+    end
+
+    feed # return new feed
+  end # method build_feed
+
+
+  def build_author( hy )
+    author = Author.new
+
+    author.name     = hy.name
+
+    ## todo - add:
+    ## author.url
+
+    author
+  end
+
+
+
+  def build_item( hy )
+    item = Item.new   # Item.new
+
+    item.title           = hy.name
+    item.url             = hy.url
+    item.published_local = hy.published_local
+    item.published       = hy.published
+
+    item.content_html    = hy.content_html
+    item.content_text    = hy.content_text
+    item.summary         = hy.summary
+
+    ##  check: how to add an id - auto-generate - why? why not??
+    ## item.id         = h['id']
+
+    hy.authors.each do |author|
+      item.authors << build_author( author )
+    end
+
+    item
+  end # method build_item
+
+end # class HyFeedBuilder
+
+
+
+class HyFeed
+  attr_accessor :entries
+
+  def initialize
+     @entries = []
+  end
+end # class HyFeed
+
+
+class HyEntry
+   attr_accessor :name
+   attr_accessor :content
+   attr_accessor :content_text
+   attr_accessor :summary
+
+   attr_accessor :published          # utc time
+   attr_accessor :published_local    # local time (with timezone/offset)
+   attr_accessor :url
+
+   attr_accessor :authors      # note: allow multiple authors
+
+   # note: title is an alias for name
+   alias :title  :name
+   alias :title= :name=
+
+   # note: content_html is an alias for name
+   alias :content_html  :content
+   alias :content_html= :content=
+
+   def initialize
+     @authors = []
+   end
+
+end  ## class HyEntry
+
+
+class HyAuthor
+   attr_accessor :name
+   attr_accessor :url
+end  ## class HyAuthor
+
+
+
+
+class HyBuilder
+
+   attr_reader :feeds
+
+   def initialize( hash )
+     @h     = hash
+     @feeds = []
+     build
+
+     pp @feeds
+   end
+
+   def build
+
+    entries = []
+    @h['items'].each_with_index do |item_hash,i|
+      puts "item #{i+1}:"
+      pp item_hash
+
+      types = item_hash['type']
+      pp types
+      if types.include?( 'h-feed' )
+        @feeds << build_feed( item_hash )
+      elsif types.include?( 'h-entry' )
+        entries << build_entry( item_hash )
+      else
+        ## unknown type; skip for now
+      end
+    end
+
+    ## wrap all "loose" entries in a "dummy" h-entry feed
+    if entries.any?
+       feed = HyFeed.new
+       feed.entries = entries
+       @feeds << feed
+    end
+
+  end # method build
+
+  def build_feed( h )
+     puts "  build_feed"
+
+     feed = HyFeed.new
+
+     h['children'].each_with_index do |item_hash,i|
+      puts "item #{i+1}:"
+      pp item_hash
+
+      types = item_hash['type']
+      pp types
+      if types.include?( 'h-entry' )
+        feed.entries << build_entry( item_hash )
+      else
+        ## unknown type; skip for now
+      end
+     end
+
+     feed
+  end  ## method build_feed
+
+
+  def build_entry( h )
+     puts "  build_entry"
+
+     entry = HyEntry.new
+
+     props = h['properties']
+     pp props
+
+     entry.name    = props['name'].join( '  ')     # check an example with more entries (how to join??)
+
+     if props['summary']
+       entry.summary = props['summary'].join( '  ' )
+     end
+
+     if props['content']
+       ## add up all value attribs in content
+       entry.content_text =  props['content'].map { |h| h[:value] }.join( '  ' ).strip
+       ## add up all html attribs in content; plus strip leading n trailing whitespaces
+       entry.content =  props['content'].map { |h| h[:html] }.join( '  ' ).strip
+     end
+
+
+     # get first field in array  -- check if really ever possible more than one? what does it mean (many dates)???
+     ##  todo: check if datetime is always utc (or local possible?)
+     url_str = props.fetch( 'url', [] )[0]
+     if url_str
+       entry.url = url_str
+     end
+
+     # get first field in array  -- check if really ever possible more than one? what does it mean (many dates)???
+     ##  todo: check if datetime is always utc (or local possible?)
+     published_str = props.fetch( 'published', [] )[0]
+     pp published_str
+     if published_str
+       ## entry.published = DateTime.iso8601( published_str )
+       entry.published_local = DateTime.parse( published_str )
+       entry.published       = entry.published_local.utc
+     end
+
+     ## check for authors
+     if props['author']
+       props['author'].each do |author_hash|
+         pp author_hash
+         entry.authors << build_author( author_hash )
+       end
+     end
+
+     entry
+  end  # method build_entry
+
+  def build_author( h )
+    puts "  build_author"
+
+    author = HyAuthor.new
+
+    author.name = h['value']
+
+    ## todo/fix: -- note: for now skip possible embedded h-card
+    author
+  end  # method build_author
+
+
+end  # class HyBuilder
+
+
+
+end # module FeedParser