feedparser 2.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12a89b6c4d0ad5290a16d44a79245a25bb6ff349
-  data.tar.gz: 63057209038f7cab23eb4fd963d9525b154d2fbb
+  metadata.gz: 417617dcf5fa45dfb199a7e692d2602b8dcb9e3f
+  data.tar.gz: 0fe1defd0d6bd634e228b067b781b21dcbb7bb3b
 SHA512:
-  metadata.gz: 6220dd036c705fedb52a17495001003c88891a912c3ad9e3fed07045adde3f0902d8dcbf3fc7000fbc297b91bba388979c54fbdaf1890f20e541d9216bfc5438
-  data.tar.gz: 049a3b9cdf4b27fe1fbc2e9109241ed951e4a717e6f620bd6aa7b2b17c9657e1fe468b86fd52cd1bf5ff48192935b3939d6579ca612c5ac576553c2f9bfcb433
+  metadata.gz: f0894102ebd9b750c12476782f2305b0d8a40946d2a209bc3e19a865ae32b9b17c210449b86f877d67922ba4ae860ba0e454d4de78b3c50b665e0933490a7b3a
+  data.tar.gz: 4b6f400c017e704d85a2ce7ec512532752a905bf15b541d452c5d32a07ec0dcb46e6b80470e5708081b0dae0d203e0a3074555294e826534fd13bef00f887f6f

data/Manifest.txt

@@ -1,8 +1,9 @@
-HISTORY.md
+CHANGELOG.md
 Manifest.txt
 README.md
 Rakefile
 lib/feedparser.rb
+lib/feedparser/attachment.rb
 lib/feedparser/author.rb
 lib/feedparser/builder/atom.rb
 lib/feedparser/builder/json.rb
@@ -13,9 +14,12 @@ lib/feedparser/generator.rb
 lib/feedparser/item.rb
 lib/feedparser/parser.rb
 lib/feedparser/tag.rb
+lib/feedparser/thumbnail.rb
 lib/feedparser/version.rb
 test/helper.rb
+test/media_rss_example.txt
 test/test_atom_live.rb
+test/test_attachments_live.rb
 test/test_dates.rb
 test/test_microformats.rb
 test/test_rss_live.rb

data/README.md

@@ -1,6 +1,6 @@
 # feedparser
 
-feedparser gem - web feed parser and normalizer (Atom, RSS 2.0, JSON Feed, HTML h-entry, etc.)
+feedparser gem - web feed parser and normalizer (Atom, RSS, JSON Feed, HTML h-entry, etc.)
 
 * home  :: [github.com/feedparser/feedparser](https://github.com/feedparser/feedparser)
 * bugs  :: [github.com/feedparser/feedparser/issues](https://github.com/feedparser/feedparser/issues)
@@ -11,6 +11,8 @@ feedparser gem - web feed parser and normalizer (Atom, RSS 2.0, JSON Feed, HTML
 
 ## What's News?
 
+**October/2017**: Added support for attachments / media enclosures in RSS and Atom.
+
 **June/2017**:  Added support for reading feeds in HTML with Microformats incl.
 [`h-entry`](http://microformats.org/wiki/h-entry),
 [`h-feed`](http://microformats.org/wiki/h-feed) and others.
@@ -32,7 +34,7 @@ See the [Awesome Feeds](https://github.com/feedparser/awesome-feeds) page ».
 
 ### Structs
 
-Feed • Item • Author • Tag • Generator
+Feed • Item • Author • Tag • Attachment • Generator
 
 
 ![](feed-models.png)
@@ -158,12 +160,20 @@ end
 
 (Source: [`lib/feedparser/tag.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/tag.rb))
 
+### `Attachment`  Struct
+
+_Also known as Media Enclosure_
+
+(Source: [`lib/feedparser/attachment.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/attachment.rb))
+
+
 ### `Generator` Struct
 
 (Source: [`lib/feedparser/generator.rb`](https://github.com/feedparser/feedparser/blob/master/lib/feedparser/generator.rb))
 
 
 
+
 ### Read Feed Example
 
 ``` ruby

data/Rakefile

@@ -5,7 +5,7 @@ Hoe.spec 'feedparser' do
 
   self.version = FeedParser::VERSION
 
-  self.summary = 'feedparser - web feed parser and normalizer (RSS 2.0, Atom, JSON Feed, HTML h-entry, etc.)'
+  self.summary = 'feedparser - web feed parser and normalizer (RSS, Atom, JSON Feed, HTML h-entry, etc.)'
   self.description = summary
 
   self.urls    = ['https://github.com/feedparser/feedparser']
@@ -15,11 +15,12 @@ Hoe.spec 'feedparser' do
 
   # switch extension to .markdown for gihub formatting
   self.readme_file  = 'README.md'
-  self.history_file = 'HISTORY.md'
+  self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
     ['logutils', '>=0.6.1'],
     ['textutils', '>=1.0.0'],
+    ## ['oga', '>=3.2.0'],   note: oga is a "soft" dependency
   ]
 
   ###  todo: add fetcher dep for testing (e.g. development only)
@@ -27,7 +28,7 @@ Hoe.spec 'feedparser' do
   self.licenses = ['Public Domain']
 
   self.spec_extras = {
-   required_ruby_version: '>= 1.9.2'
+   required_ruby_version: '>= 2.2.2'
   }
 
 end

data/lib/feedparser.rb

@@ -30,6 +30,8 @@ require 'feedparser/feed'
 require 'feedparser/item'
 require 'feedparser/author'
 require 'feedparser/tag'
+require 'feedparser/attachment'
+require 'feedparser/thumbnail'
 require 'feedparser/generator'
 require 'feedparser/parser'
 

data/lib/feedparser/attachment.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+
+module FeedParser
+
+class Attachment   ## also known as Enclosure
+
+  attr_accessor :url
+  ## note: uri is an alias for url
+  alias :uri  :url       ## add atom alias for uri - why? why not?
+  alias :uri= :url=
+
+  attr_accessor :length
+  attr_accessor :type
+
+  # Elements from the media namespace attachment
+  attr_accessor :title
+  attr_accessor :thumbnail
+  attr_accessor :description
+  attr_accessor :community
+
+end  # class Attachment
+
+end # module FeedParser

data/lib/feedparser/author.rb

@@ -22,7 +22,7 @@ class Author
   ##          or managingEditor (rss) or webMaster (rss) - why? why not??
 
   attr_accessor :text    # note: holds "unparsed" text (content) line form dc:creator or rss:author
-
+  alias :line :text     # line|text  (add str??  too)
 
   def to_s
     ## note: to_s  - allows to use just author in templates

data/lib/feedparser/builder/atom.rb

@@ -7,13 +7,13 @@ class AtomFeedBuilder
   include LogUtils::Logging
 
 
-  def self.build( atom_feed )
-    feed = self.new( atom_feed )
+  def self.build( atom_feed, raw )
+    feed = self.new( atom_feed, raw )
     feed.to_feed
   end
 
-  def initialize( atom_feed )
-    @feed = build_feed( atom_feed )
+  def initialize( atom_feed, raw )
+    @feed = build_feed( atom_feed, raw )
   end
 
   def to_feed
@@ -22,7 +22,7 @@ class AtomFeedBuilder
 
 
 
-  def build_feed( atom_feed )    ## fix/todo: rename atom_feed to atom or wire or xml or in ???
+  def build_feed( atom_feed, raw )    ## fix/todo: rename atom_feed to atom or wire or xml or in ???
     feed = Feed.new
     feed.format = 'atom'
 
@@ -110,6 +110,16 @@ class AtomFeedBuilder
       feed.items << build_item( atom_item )
     end
 
+
+    if defined?( Oga )
+      # Use Oga as generic xml parser to access elements not adressed by the core RSS module like media:
+      parsed_xml = Oga.parse_xml( raw )
+      xml_items = parsed_xml.xpath( '/feed/entry' )
+      xml_items.each_with_index do |xml_item, i|
+          feed.items[i] = add_meta_items( feed.items[i], xml_item )
+      end
+    end
+
     feed # return new feed
   end # method build_feed_from_atom
 
@@ -176,7 +186,7 @@ class AtomFeedBuilder
       item.updated        = item.updated_local.utc
     end
 
-    if atom_item.published && atom_item.published.content 
+    if atom_item.published && atom_item.published.content
       item.published_local  = handle_date( atom_item.published, 'item.published' )
       item.published        = item.published_local.utc
     end
@@ -203,10 +213,54 @@ class AtomFeedBuilder
       item.tags << build_tag( atom_cat )
     end
 
+
+    ## check for attachments / media enclosures
+    ###  todo/fix: allow more than one attachment/enclosure
+    if atom_item.links
+      enclosure = atom_item.links.detect{ |x| x.rel == 'enclosure' }
+      if enclosure
+        attachment = Attachment.new
+        attachment.url    = enclosure.href
+        attachment.length = enclosure.length
+        attachment.type   = enclosure.type
+        item.attachments << attachment
+      end
+    end
+
     item
   end # method build_item
 
 
+  # Add additional elements, currently the media: namespace elements
+  # Note: This tries to accomodate both the different ways to transport the data via the spec https://www.rssboard.org/media-rss/ and the practice by Youtube of grouping everything under media:group
+  def add_meta_items( feed_item, xml_item )
+    if xml_item.at_xpath('media:group') || xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content') || xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:description')
+      feed_item.attachments << Attachment.new unless feed_item.attachments.first
+
+      titleElement = xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content/media:title') || xml_item.at_xpath('media:group/media:title')
+      feed_item.attachments.first.title = titleElement.text if titleElement
+
+      contentElement = xml_item.at_xpath('media:content') || xml_item.at_xpath('media:group/media:content')
+      if contentElement
+        feed_item.attachments.first.url = contentElement.get('url')
+        feed_item.attachments.first.length = contentElement.get('duration')
+      end
+
+      thumbnailElement = xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:content/media:thumbnail') || xml_item.at_xpath('media:group/media:thumbnail')
+      if thumbnailElement
+        thumbnail = Thumbnail.new
+        thumbnail.url = thumbnailElement.get('url')
+        thumbnail.width = thumbnailElement.get('width')
+        thumbnail.height = thumbnailElement.get('height')
+        feed_item.attachments.first.thumbnail = thumbnail
+      end
+
+      descriptionElement = xml_item.at_xpath('media:description') || xml_item.at_xpath('media:content/media:description') || xml_item.at_xpath('media:group/media:description')
+      feed_item.attachments.first.description = descriptionElement.text if descriptionElement
+    end
+    feed_item
+  end # method add_meta_items
+
 
   def handle_date( el, name )
     ## change time to utc if present? why? why not?

data/lib/feedparser/builder/rss.rb

@@ -10,13 +10,13 @@ class RssFeedBuilder
   include LogUtils::Logging
 
 
-  def self.build( rss_feed )
-    feed = self.new( rss_feed )
+  def self.build( rss_feed, raw )
+    feed = self.new( rss_feed, raw )
     feed.to_feed
   end
 
-  def initialize( rss_feed )
-    @feed = build_feed( rss_feed )
+  def initialize( rss_feed, raw )
+    @feed = build_feed( rss_feed, raw )
   end
 
   def to_feed
@@ -25,7 +25,7 @@ class RssFeedBuilder
 
 
 
-  def build_feed( rss_feed )
+  def build_feed( rss_feed, raw )
     feed = Feed.new
     feed.format = "rss #{rss_feed.rss_version}"
 
@@ -35,29 +35,42 @@ class RssFeedBuilder
     feed.summary   = handle_content( rss_feed.channel.description, 'feed.description => summary' )  # required
     feed.url       = rss_feed.channel.link            # required
 
-    feed.updated_local   = handle_date( rss_feed.channel.lastBuildDate, 'feed.lastBuildDate => updated' )  # optional
+    begin
+        feed.updated_local   = handle_date( rss_feed.channel.lastBuildDate, 'feed.lastBuildDate => updated' )  # optional
+    rescue
+    end
     feed.updated         = feed.updated_local.utc     if feed.updated_local
 
-    feed.published_local = handle_date( rss_feed.channel.pubDate,       'feed.pubDate => published'     )  # optional
+    begin
+        feed.published_local = handle_date( rss_feed.channel.pubDate,       'feed.pubDate => published'     )  # optional
+    rescue
+    end
     feed.published       = feed.published_local.utc   if feed.published_local
 
-    logger.debug "  rss | feed.generator  >#{rss_feed.channel.generator}< : #{rss_feed.channel.generator.class.name}"
+    begin
+        logger.debug "  rss | feed.generator  >#{rss_feed.channel.generator}< : #{rss_feed.channel.generator.class.name}"
+    rescue
+    end
 
-    feed.generator.text = rss_feed.channel.generator    # optional
+    begin
+        feed.generator.text = rss_feed.channel.generator    # optional
+    rescue
+    end
     feed.generator.name = feed.generator.text   ## note: for now set also name/title to "unparsed" (content) line (may change in the future!!!)
 
 
 
     ## check for managingEditor and/or  webMaster
 
-    if rss_feed.channel.managingEditor
+    if rss_feed.channel.respond_to?(:managingEditor) && rss_feed.channel.managingEditor
       author = Author.new
       author.text = rss_feed.channel.managingEditor.strip
       author.name = author.text   ## note: for now use "unparsed" (content) line also for name
       feed.authors << author
     end
 
-    if rss_feed.channel.webMaster
+    ## todo/check - if tag is called webmaster or webMaster ???
+    if rss_feed.channel.respond_to?(:webMaster) && rss_feed.channel.webMaster
       author = Author.new
       author.text = rss_feed.channel.webMaster.strip
       author.name = author.text   ## note: for now use "unparsed" (content) line also for name
@@ -76,9 +89,10 @@ class RssFeedBuilder
 
 
     ###  check for categories (tags)
-
-    rss_feed.channel.categories.each do |rss_cat|
-      feed.tags << build_tag( rss_cat )
+    if rss_feed.channel.respond_to?(:categories)
+        rss_feed.channel.categories.each do |rss_cat|
+          feed.tags << build_tag( rss_cat )
+        end
     end
 
 
@@ -86,6 +100,14 @@ class RssFeedBuilder
       feed.items << build_item( rss_item )
     end
 
+    if defined?( Oga )
+      parsed_xml = Oga.parse_xml( raw )
+      xml_items = parsed_xml.xpath( '/rss/channel/item' )
+      xml_items.each_with_index do |xml_item, i|
+        feed.items[i] = add_meta_items( feed.items[i], xml_item )
+      end
+    end
+
     feed # return new feed
   end
 
@@ -138,16 +160,18 @@ class RssFeedBuilder
     item.content = rss_item.content_encoded
     logger.debug "  rss | item.content_encoded[0..40]  >#{rss_item.content_encoded ? rss_item.content_encoded[0..40] : ''}< : #{rss_item.content_encoded.class.name}"
 
-
-    item.updated_local   = handle_date( rss_item.pubDate, 'item.pubDate => updated' )
-    item.updated         = item.updated_local.utc    if item.updated_local
+    begin
+        item.published_local   = handle_date( rss_item.pubDate, 'item.pubDate => published' )
+    rescue
+    end
+    item.published         = item.published_local.utc    if item.published_local
 
 
     ## fix/todo: check if rss_item.guid present? !!!!
     ##
     ##  might be the case e.g. check lambda-the-ultimate.org, for example
 
-    if rss_item.guid && rss_item.guid.content
+    if rss_item.respond_to?(:guid) && rss_item.guid && rss_item.guid.content
       item.guid     = rss_item.guid.content
       logger.debug "  rss | item.guid.content  >#{rss_item.guid.content}< : #{rss_item.guid.content.class.name}"
     else
@@ -156,7 +180,7 @@ class RssFeedBuilder
     end
 
 
-    if rss_item.author
+    if rss_item.respond_to?(:author) && rss_item.author
       author = Author.new
       author.text = rss_item.author.strip
       author.name = author.text   ## note: for now use "unparsed" (content) line also for name
@@ -173,18 +197,68 @@ class RssFeedBuilder
       item.authors = authors
     end
 
+    unless item.published_local
+        # use dc_date only of no regular item date was given
+        begin
+            item.published_local   = handle_date( rss_item.dc_date, 'item.dc_date => published' )
+        rescue
+        end
+        item.published         = item.published_local.utc    if item.published_local
+    end
 
     ###  check for categories (tags)
-
-    rss_item.categories.each do |rss_cat|
-      item.tags << build_tag( rss_cat )
+    if rss_item.respond_to?(:categories)
+        rss_item.categories.each do |rss_cat|
+          item.tags << build_tag( rss_cat )
+        end
     end
 
 
+    ## check for enclosure
+    ##   todo/check: rss can only include at most one enclosure?
+
+    if rss_item.respond_to?(:enclosure) && rss_item.enclosure
+      attachment = Attachment.new
+      attachment.url    = rss_item.enclosure.url
+      attachment.length = rss_item.enclosure.length
+      attachment.type   = rss_item.enclosure.type
+      item.attachments << attachment
+    end
+
     item
   end # method build_feed_item_from_rss
 
 
+  # Add additional elements, currently the media: namespace elements
+  def add_meta_items( feed_item, xml_item )
+    if xml_item.at_xpath('media:group') || xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content') || xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:description')
+      feed_item.attachments << Attachment.new unless feed_item.attachments.first
+
+      titleElement = xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content/media:title') || xml_item.at_xpath('media:group/media:title')
+      feed_item.attachments.first.title = titleElement.text if titleElement
+
+      contentElement = xml_item.at_xpath('media:content') || xml_item.at_xpath('media:group/media:content')
+      if contentElement
+        feed_item.attachments.first.url = contentElement.get('url')
+        feed_item.attachments.first.length = contentElement.get('duration')
+      end
+
+      thumbnailElement = xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:content/media:thumbnail') || xml_item.at_xpath('media:group/media:thumbnail')
+      if thumbnailElement
+        thumbnail = Thumbnail.new
+        thumbnail.url = thumbnailElement.get('url')
+        thumbnail.width = thumbnailElement.get('width')
+        thumbnail.height = thumbnailElement.get('height')
+        feed_item.attachments.first.thumbnail = thumbnail
+      end
+
+      descriptionElement = xml_item.at_xpath('media:description') || xml_item.at_xpath('media:content/media:description') || xml_item.at_xpath('media:group/media:description')
+      feed_item.attachments.first.description = descriptionElement.text if descriptionElement
+    end
+    feed_item
+  end # method add_meta_items
+
+
 
   def handle_date( el, name )
     ## change time to utc if present? why? why not?

data/lib/feedparser/feed.rb

@@ -25,10 +25,19 @@ class Feed
   attr_accessor :tags
   def tags?()  @tags && @tags.size > 0;  end
 
+  ## add alias category for tags (remove - why? why not?)
+  alias :categories :tags
+
 
   def summary?()  @summary.nil? == false;  end
   attr_accessor :summary        # e.g. description (rss)|subtitle (atom)
 
+  ## add description as alias for summary (remove - why? why not?)
+  alias :description  :summary
+  alias :description= :summary=
+  alias :description? :summary?
+
+
   ##
   ##  todo/check/fix:
   ##     use a extra field for atom subtitle
@@ -47,10 +56,16 @@ class Feed
   attr_accessor :updated        # e.g. lastBuildDate (rss)|updated (atom)   -- always (converted) to utc
   attr_accessor :updated_local  # "unparsed" local datetime as in feed (NOT converted to utc)
 
+  attr_accessor :updated_text    #  string version of date
+  alias :updated_line :updated_text   # text|line - convention for "unparsed" 1:1 from feed; add str(too ??)
+
   def published?()  @published.nil? == false;  end
   attr_accessor :published         # e.g. pubDate (rss)\n/a (atom)  -- note: published is basically an alias for created
   attr_accessor :published_local   # "unparsed" local datetime as in feed (NOT converted to utc)
 
+  attr_accessor :published_text    #  string version of date
+  alias :published_line :published_text   # text|line - convention for "unparsed" 1:1 from feed; add str(too ??)
+
 
   attr_accessor :generator
 

data/lib/feedparser/generator.rb

@@ -18,6 +18,8 @@ class Generator
 
 
   attr_accessor :text  # note: holds "unparsed" text (content) line form rss:generator
+  alias :line :text    # line|text (add str?? too)
+
 
   def to_s
     ## note: to_s  - allows to use just generator in templates

data/lib/feedparser/item.rb

@@ -35,15 +35,27 @@ class Item
   def summary?()  @summary.nil? == false;  end
   attr_accessor   :summary
 
+  ## add description as alias for summary (remove - why? why not?)
+  alias :description  :summary
+  alias :description= :summary=
+  alias :description? :summary?
+
+
 
   def updated?()  @updated.nil? == false;  end
   attr_accessor :updated   # pubDate (RSS)|updated (Atom)
   attr_accessor :updated_local  # "unparsed" local datetime as in feed (NOT converted to utc)
 
+  attr_accessor :updated_text    #  string version of date
+  alias :updated_line :updated_text   # text|line - convention for "unparsed" 1:1 from feed; add str(too ??)
+
+
   def published?()  @published.nil? == false;  end
   attr_accessor :published  # note: published is basically an alias for created
   attr_accessor :published_local   # "unparsed" local datetime as in feed (NOT converted to utc)
 
+  attr_accessor :published_text    #  string version of date
+  alias :published_line :published_text   # text|line - convention for "unparsed" 1:1 from feed; add str(too ??)
 
 
   attr_accessor :id
@@ -67,10 +79,28 @@ class Item
   attr_accessor :tags
   def tags?()  @tags && @tags.size > 0;  end
 
+  alias :categories :tags    # for now allow categories alias for tags - remove (why? why not?)
+
+
+  # add attachments/media enclosures (url, length and type)
+  #  note: lets support more than one (it's an array)
+  attr_accessor :attachments
+
+  def attachment()    @attachments[0]; end
+  def attachments?()  @attachments && @attachments.size > 0;  end
+  alias :attachment? :attachments?
+
+  alias :enclosures  :attachments
+  alias :enclosure   :attachment
+  alias :enclosures? :attachments?
+  alias :enclosure?  :attachments?
+
+
   def initialize
     ## note: make authors, tags empty arrays on startup (e.g. not nil)
-    @authors = []
-    @tags    = []
+    @authors     = []
+    @tags        = []
+    @attachments = []
   end
 
 end  # class Item

data/lib/feedparser/parser.rb

@@ -16,38 +16,53 @@ class Parser
   ### Note: lets keep/use same API as RSS::Parser for now
   def initialize( text )
     @text = text
+    @head = @text[0..100].strip     # note: remove leading spaces if present
   end
 
 
 
-  def parse
-    head = @text[0..100].strip     # note: remove leading spaces if present
-
-    jsonfeed_version_regex = %r{"version":\s*"https://jsonfeed.org/version/1"}
+  #### note:
+  # make format checks callable from outside (that is, use builtin helper methods)
 
+  def is_xml?
     ## check if starts with knownn xml prologs
-    if head.start_with?( '<?xml' )  ||
-       head.start_with?( '<feed/' ) ||
-       head.start_with?( '<rss/' )
+    @head.start_with?( '<?xml' )  ||
+    @head.start_with?( '<feed' ) ||
+    @head.start_with?( '<rss' )
+  end
+  alias_method :xml?, :is_xml?
+
+  JSONFEED_VERSION_RE = %r{"version":\s*"https://jsonfeed.org/version/1"}
+  def is_json?
     ## check if starts with { for json object/hash
     ##    or if includes jsonfeed prolog
+    @head.start_with?( '{' ) ||
+    @head =~ JSONFEED_VERSION_RE
+  end
+  alias_method :json?, :is_json?
+
+  def is_microformats?
+    #  for now check for microformats v2 (e.g. h-entry, h-feed)
+    #    check for v1 too - why? why not? (e.g. hentry, hatom ??)
+    @text.include?( 'h-entry' ) ||
+    @text.include?( 'h-feed' )
+  end
+  alias_method :microformats?, :is_microformats?
+
+
+
+  def parse
+    if is_xml?
        parse_xml
-    elsif head.start_with?( '{' ) ||
-          head =~ jsonfeed_version_regex
+    elsif is_json?
        parse_json
     ##  note: reading/parsing microformat is for now optional
     ##    microformats gem requires nokogiri
     ##       nokogiri (uses libxml c-extensions) makes it hard to install (sometime)
     ##       thus, if you want to use it, please opt-in to keep the install "light"
-    #
-    #  for now check for microformats v2 (e.g. h-entry, h-feed)
-    #    check for v1 too - why? why not? (e.g. hentry, hatom ??)
-  elsif defined?( Microformats ) &&
-          (@text.include?( 'h-entry' ) ||
-           @text.include?( 'h-feed' )
-           )
-      parse_microformats
-    else  ## assume xml for now
+    elsif defined?( Microformats ) && is_microformats?
+       parse_microformats
+    else  ## fallback - assume xml for now
        parse_xml
     end
   end # method parse
@@ -95,9 +110,9 @@ class Parser
     logger.debug "  feed.class=#{feed_wild.class.name}"
 
     if feed_wild.is_a?( RSS::Atom::Feed )
-      feed = AtomFeedBuilder.build( feed_wild )
+      feed = AtomFeedBuilder.build( feed_wild, @text )
     else  # -- assume RSS::Rss::Feed
-      feed = RssFeedBuilder.build( feed_wild )
+      feed = RssFeedBuilder.build( feed_wild, @text )
     end
 
     logger.debug "== #{feed.format} / #{feed.title} =="

data/lib/feedparser/thumbnail.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+module FeedParser
+
+class Thumbnail
+
+  attr_accessor :url
+  
+  ## note: uri is an alias for url
+  alias :uri  :url       ## add atom alias for uri - why? why not?
+  alias :uri= :url=
+
+  def width?()   @width.nil? == false;  end
+  attr_accessor :width
+
+  def height?()  @height.nil? == false;  end
+  attr_accessor :height  # todo/check: use avatar_url ?? used by json feed -check if always a url
+
+end  # class Thumbnail
+
+end # module FeedParser

data/lib/feedparser/version.rb

@@ -3,7 +3,7 @@
 module FeedParser
 
   MAJOR = 2
-  MINOR = 0
+  MINOR = 2
   PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 

data/test/media_rss_example.txt

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/" version="2.0">
+   <channel>
+      <title>Calm Meditation</title>
+      <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com</link>
+      <language>en-us</language>
+      <pubDate>Mon, 02 Apr 2018 16:19:56 -0700</pubDate>
+      <lastBuildDate>Mon, 02 Apr 2018 16:19:56 -0700</lastBuildDate>
+      <managingEditor>tomjoht@gmail.com (Tom Johnson)</managingEditor>
+      <description>Contains short videos capturing still scenes from nature with a music background, intended for calming or meditation purposes. When you're stressed out or upset, watch a few videos. As your mind focuses on the small details, let your worries and frustrations float away. The purpose is not to entertain or to distract, but to help calm, soothe, and surface your inner quiet. The videos contain scenes from the San Tomas Aquinas trail in Santa Clara, California.</description>
+      <image>
+         <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com</link>
+         <title>Calm Meditation</title>
+         <url>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/images/calmmeditationlogo_small.png</url>
+         <description>Contains short videos capturing still scenes from nature with a music background, intended for calming or meditation purposes. When you're stressed out or upset, watch a few videos. As your mind focuses on the small details, let your worries and frustrations float away. The purpose is not to entertain or to distract, but to help calm, soothe, and surface your inner quiet. The videos contain scenes from the San Tomas Aquinas trail in Santa Clara, California.</description>
+         <height>114</height>
+         <width>114</width>
+      </image>
+      <atom:link href="http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/feed.xml" rel="self" type="application/rss+xml" />
+      <item>
+         <title>Shade</title>
+         <pubDate>Mon, 23 Oct 2017 00:00:00 -0700</pubDate>
+         <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/shade/</link>
+         <description>Quiet the mind, and the soul will speak. - Ma Jaya Sati Bhagavati</description>
+         <guid isPermaLink="false">http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/shade/</guid>
+         <media:category>All</media:category>
+         <media:category>Trail</media:category>
+         <media:content url="http://d1nixf144dcz0j.cloudfront.net/shade.mp4" language="en-us" fileSize="37000000" duration="120.0" medium="video" isDefault="true">
+            <media:title type="plain">Shade</media:title>
+            <media:description type="html">Quiet the mind, and the soul will speak. - Ma Jaya Sati Bhagavati</media:description>
+            <media:thumbnail url="http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/images/thumbs/shade.jpg" />
+            <media:credit role="author" scheme="urn:ebu">Tom Johnson</media:credit>
+            <media:copyright url="https://creativecommons.org/licenses/by/4.0/" />
+         </media:content>
+      </item>
+      <item>
+         <title>Spectators</title>
+         <pubDate>Thu, 12 Oct 2017 00:00:00 -0700</pubDate>
+         <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/spectators/</link>
+         <description>"Your worst enemy cannot harm you as much as your own thoughts, unguarded." – Buddha</description>
+         <guid isPermaLink="false">http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/spectators/</guid>
+         <media:category>All</media:category>
+         <media:category>Grass</media:category>
+         <media:content url="http://d1nixf144dcz0j.cloudfront.net/spectators.mp4" language="en-us" fileSize="19000000" duration="120.0" medium="video" isDefault="true">
+            <media:title type="plain">Spectators</media:title>
+            <media:description type="html">"Your worst enemy cannot harm you as much as your own thoughts, unguarded." – Buddha</media:description>
+            <media:thumbnail url="http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/images/thumbs/spectators.jpg" />
+            <media:credit role="author" scheme="urn:ebu">Tom Johnson</media:credit>
+            <media:copyright url="https://creativecommons.org/licenses/by/4.0/" />
+         </media:content>
+      </item>
+   </channel>
+</rss>

data/test/test_atom_live.rb

@@ -4,6 +4,8 @@
 #  or better
 #     rake test
 
+
+
 require 'helper'
 
 class TestAtomLive < MiniTest::Test
@@ -11,36 +13,48 @@ class TestAtomLive < MiniTest::Test
   def test_rubyonrails
     feed = fetch_and_parse_feed( 'http://weblog.rubyonrails.org/feed/atom.xml' )
 
-    assert_equal 'atom',                           feed.format
-    assert_equal 'http://weblog.rubyonrails.org/', feed.url
+    assert_equal 'atom',                            feed.format
+    assert_equal 'https://weblog.rubyonrails.org/', feed.url
+    ## note was (2020/1): 'http://weblog.rubyonrails.org/', feed.url
   end
 
 
   def test_railstutorial
     feed = fetch_and_parse_feed( 'http://feeds.feedburner.com/railstutorial?format=xml' )
 
-    assert_equal 'atom',                           feed.format
-    assert_equal 'http://news.learnenough.com/',   feed.url
+    assert_equal 'atom',                            feed.format
+    assert_equal 'https://news.learnenough.com/',   feed.url
+    ## note was (2020/1):  assert_equal 'http://news.learnenough.com/',   feed.url
     ## note was (2017/5):  assert_equal 'http://news.railstutorial.org/', feed.url
   end
 
 
+=begin
+  ### returns ssl error e.g.
+  ## OpenSSL::SSL::SSLError: SSL_connect SYSCALL returned=5 errno=0 state=SSLv2/v3 read server
   def test_googlegroup
     feed = fetch_and_parse_feed( 'https://groups.google.com/forum/feed/beerdb/topics/atom.xml?num=15' )
 
     assert_equal 'atom', feed.format
     assert_equal 'https://groups.google.com/d/forum/beerdb', feed.url
   end
+=end
 
 
   def test_headius
-    feed = fetch_and_parse_feed( 'http://blog.headius.com/feeds/posts/default' )
+    feed = fetch_and_parse_feed( 'http://blog.headius.com/feed.xml' )
+    ## note was (2020/1): 'http://blog.headius.com/feeds/posts/default'
 
     assert_equal 'atom',    feed.format
-    assert_equal 'Blogger', feed.generator.name
-    assert_equal 'Headius', feed.title
-    assert_equal 'Helping the JVM Into the 21st Century', feed.summary  # aka subtitle in atom
-    assert_equal 'http://blog.headius.com/', feed.url
+    assert_equal 'Jekyll',  feed.generator.name
+    ## note was (2020/1): 'Blogger'
+    
+    assert_equal 'Charles Oliver Nutter', feed.title
+    ## note was (2020/1): 'Headius', feed.title
+    assert_equal 'Java, Ruby, and JVM guy trying to make sense of it all', feed.summary  # aka subtitle in atom
+    ## note was (2020/1): 'Helping the JVM Into the 21st Century', feed.title
+    assert_equal 'https://headius.github.io/', feed.url
+    ## note was (2020/1): 'http://blog.headius.com/' 
   end
 
 end

data/test/test_attachments_live.rb

@@ -0,0 +1,69 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_attachments_live.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+
+###
+## note: needs to require oga gem (it's not required by default - it's a "soft" dependency)
+
+require 'oga'
+
+
+
+class TestAttachmentsLive < MiniTest::Test
+
+  def test_atom_enclose
+    feed = fetch_and_parse_feed( 'http://www.lse.ac.uk/assets/richmedia/webFeeds/publicLecturesAndEvents_AtomAllMediaTypesLatest100.xml' )
+
+    assert_equal 'audio/mpeg', feed.items.first.attachment.type
+    assert_equal 'audio/mpeg', feed.items.first.enclosure.type
+
+    assert_equal true, feed.items.first.attachment?
+    assert_equal true, feed.items.first.enclosure?
+  end
+
+  def test_atom_media
+    feed = fetch_and_parse_feed( 'http://www.youtube.com/feeds/videos.xml?channel_id=UCZUT79WUUpZlZ-XMF7l4CFg' )
+    assert_equal true, feed.items.first.attachment?
+    assert feed.items.first.attachments.first.title
+    assert feed.items.first.attachments.first.url
+    assert feed.items.first.attachments.first.thumbnail
+    assert_instance_of FeedParser::Thumbnail, feed.items.first.attachments.first.thumbnail
+    assert feed.items.first.attachments.first.thumbnail.url
+    assert_equal 480, feed.items.first.attachments.first.thumbnail.width.to_i
+    assert_equal 360, feed.items.first.attachments.first.thumbnail.height.to_i
+    assert feed.items.first.attachments.first.description
+  end
+
+  def test_rss_media
+    # tests an example RSS file from https://creator.amazon.com/documentation/ac/mrss.html. Not that unlike the Atom example, it does
+    # does not put everything under media:group
+    testpath = File.join(File.expand_path(File.dirname(__FILE__)), 'media_rss_example.txt')
+    feed_rss = File.read( testpath )
+    feed = FeedParser::Parser.parse( feed_rss )
+    assert_equal true, feed.items.first.attachment?
+    assert feed.items.first.attachments.first.title
+    assert feed.items.first.attachments.first.url
+    assert feed.items.first.attachments.first.thumbnail
+    assert_instance_of FeedParser::Thumbnail, feed.items.first.attachments.first.thumbnail
+    assert feed.items.first.attachments.first.thumbnail.url
+    assert_nil feed.items.first.attachments.first.thumbnail.width
+    assert_nil feed.items.first.attachments.first.thumbnail.height
+    assert feed.items.first.attachments.first.description
+  end
+
+  def test_rss_enclosure
+    feed = fetch_and_parse_feed( 'http://www.radiofreesatan.com/category/featured/feed/' )
+
+    assert_equal 'audio/mpeg', feed.items.first.attachment.type
+    assert_equal 'audio/mpeg', feed.items.first.enclosure.type
+
+    assert_equal true, feed.items.first.attachment?
+    assert_equal true, feed.items.first.enclosure?
+  end
+
+end

data/test/test_rss_live.rb

@@ -31,7 +31,7 @@ class TestRssLive < MiniTest::Test
   def test_rubymine
     # includes item/content:encoded
     feed = fetch_and_parse_feed( 'http://feeds.feedburner.com/jetbrains_rubymine?format=xml' )
-    
+
     assert_equal 'rss 2.0', feed.format
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedparser
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-08 00:00:00.000000000 Z
+date: 2020-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
@@ -58,29 +58,30 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
-description: feedparser - web feed parser and normalizer (RSS 2.0, Atom, JSON Feed,
-  HTML h-entry, etc.)
+        version: '3.16'
+description: feedparser - web feed parser and normalizer (RSS, Atom, JSON Feed, HTML
+  h-entry, etc.)
 email: wwwmake@googlegroups.com
 executables: []
 extensions: []
 extra_rdoc_files:
-- HISTORY.md
+- CHANGELOG.md
 - Manifest.txt
 - README.md
 files:
-- HISTORY.md
+- CHANGELOG.md
 - Manifest.txt
 - README.md
 - Rakefile
 - lib/feedparser.rb
+- lib/feedparser/attachment.rb
 - lib/feedparser/author.rb
 - lib/feedparser/builder/atom.rb
 - lib/feedparser/builder/json.rb
@@ -91,9 +92,12 @@ files:
 - lib/feedparser/item.rb
 - lib/feedparser/parser.rb
 - lib/feedparser/tag.rb
+- lib/feedparser/thumbnail.rb
 - lib/feedparser/version.rb
 - test/helper.rb
+- test/media_rss_example.txt
 - test/test_atom_live.rb
+- test/test_attachments_live.rb
 - test/test_dates.rb
 - test/test_microformats.rb
 - test/test_rss_live.rb
@@ -111,7 +115,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: 2.2.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -119,9 +123,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.7
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
-summary: feedparser - web feed parser and normalizer (RSS 2.0, Atom, JSON Feed, HTML
-  h-entry, etc.)
+summary: feedparser - web feed parser and normalizer (RSS, Atom, JSON Feed, HTML h-entry,
+  etc.)
 test_files: []