feedparser 2.1.2 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fdd14a8952605dcb40547f1675bc234fb16af21f
-  data.tar.gz: de37586ae452d80ebc2042ebfda9c29dabfa2aa0
+  metadata.gz: 417617dcf5fa45dfb199a7e692d2602b8dcb9e3f
+  data.tar.gz: 0fe1defd0d6bd634e228b067b781b21dcbb7bb3b
 SHA512:
-  metadata.gz: b79665575a1596f1c644d48307e5660a52cf6ac1b0124bf6bc5df4ce66123cabada4dacebcf9240665dc2e0d5e2e52d14d764e9183bcf5bf25d9766385996b73
-  data.tar.gz: 8cb7a605620667f035a635826c29690835a4b38fa50e932d6b2fa6ac054d34f9c37a10585b5d9fcf0438836e3462081f85bb24a67ca2c1baa04fa4659cbd59eb
+  metadata.gz: f0894102ebd9b750c12476782f2305b0d8a40946d2a209bc3e19a865ae32b9b17c210449b86f877d67922ba4ae860ba0e454d4de78b3c50b665e0933490a7b3a
+  data.tar.gz: 4b6f400c017e704d85a2ce7ec512532752a905bf15b541d452c5d32a07ec0dcb46e6b80470e5708081b0dae0d203e0a3074555294e826534fd13bef00f887f6f

data/Manifest.txt

@@ -14,8 +14,10 @@ lib/feedparser/generator.rb
 lib/feedparser/item.rb
 lib/feedparser/parser.rb
 lib/feedparser/tag.rb
+lib/feedparser/thumbnail.rb
 lib/feedparser/version.rb
 test/helper.rb
+test/media_rss_example.txt
 test/test_atom_live.rb
 test/test_attachments_live.rb
 test/test_dates.rb

data/Rakefile

@@ -20,6 +20,7 @@ Hoe.spec 'feedparser' do
   self.extra_deps = [
     ['logutils', '>=0.6.1'],
     ['textutils', '>=1.0.0'],
+    ## ['oga', '>=3.2.0'],   note: oga is a "soft" dependency
   ]
 
   ###  todo: add fetcher dep for testing (e.g. development only)

data/lib/feedparser.rb

@@ -31,6 +31,7 @@ require 'feedparser/item'
 require 'feedparser/author'
 require 'feedparser/tag'
 require 'feedparser/attachment'
+require 'feedparser/thumbnail'
 require 'feedparser/generator'
 require 'feedparser/parser'
 

data/lib/feedparser/attachment.rb

@@ -12,6 +12,12 @@ class Attachment   ## also known as Enclosure
   attr_accessor :length
   attr_accessor :type
 
+  # Elements from the media namespace attachment
+  attr_accessor :title
+  attr_accessor :thumbnail
+  attr_accessor :description
+  attr_accessor :community
+
 end  # class Attachment
 
 end # module FeedParser

data/lib/feedparser/builder/atom.rb

@@ -7,13 +7,13 @@ class AtomFeedBuilder
   include LogUtils::Logging
 
 
-  def self.build( atom_feed )
-    feed = self.new( atom_feed )
+  def self.build( atom_feed, raw )
+    feed = self.new( atom_feed, raw )
     feed.to_feed
   end
 
-  def initialize( atom_feed )
-    @feed = build_feed( atom_feed )
+  def initialize( atom_feed, raw )
+    @feed = build_feed( atom_feed, raw )
   end
 
   def to_feed
@@ -22,7 +22,7 @@ class AtomFeedBuilder
 
 
 
-  def build_feed( atom_feed )    ## fix/todo: rename atom_feed to atom or wire or xml or in ???
+  def build_feed( atom_feed, raw )    ## fix/todo: rename atom_feed to atom or wire or xml or in ???
     feed = Feed.new
     feed.format = 'atom'
 
@@ -110,6 +110,16 @@ class AtomFeedBuilder
       feed.items << build_item( atom_item )
     end
 
+
+    if defined?( Oga )
+      # Use Oga as generic xml parser to access elements not adressed by the core RSS module like media:
+      parsed_xml = Oga.parse_xml( raw )
+      xml_items = parsed_xml.xpath( '/feed/entry' )
+      xml_items.each_with_index do |xml_item, i|
+          feed.items[i] = add_meta_items( feed.items[i], xml_item )
+      end
+    end
+
     feed # return new feed
   end # method build_feed_from_atom
 
@@ -221,6 +231,36 @@ class AtomFeedBuilder
   end # method build_item
 
 
+  # Add additional elements, currently the media: namespace elements
+  # Note: This tries to accomodate both the different ways to transport the data via the spec https://www.rssboard.org/media-rss/ and the practice by Youtube of grouping everything under media:group
+  def add_meta_items( feed_item, xml_item )
+    if xml_item.at_xpath('media:group') || xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content') || xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:description')
+      feed_item.attachments << Attachment.new unless feed_item.attachments.first
+
+      titleElement = xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content/media:title') || xml_item.at_xpath('media:group/media:title')
+      feed_item.attachments.first.title = titleElement.text if titleElement
+
+      contentElement = xml_item.at_xpath('media:content') || xml_item.at_xpath('media:group/media:content')
+      if contentElement
+        feed_item.attachments.first.url = contentElement.get('url')
+        feed_item.attachments.first.length = contentElement.get('duration')
+      end
+
+      thumbnailElement = xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:content/media:thumbnail') || xml_item.at_xpath('media:group/media:thumbnail')
+      if thumbnailElement
+        thumbnail = Thumbnail.new
+        thumbnail.url = thumbnailElement.get('url')
+        thumbnail.width = thumbnailElement.get('width')
+        thumbnail.height = thumbnailElement.get('height')
+        feed_item.attachments.first.thumbnail = thumbnail
+      end
+
+      descriptionElement = xml_item.at_xpath('media:description') || xml_item.at_xpath('media:content/media:description') || xml_item.at_xpath('media:group/media:description')
+      feed_item.attachments.first.description = descriptionElement.text if descriptionElement
+    end
+    feed_item
+  end # method add_meta_items
+
 
   def handle_date( el, name )
     ## change time to utc if present? why? why not?

data/lib/feedparser/builder/rss.rb

@@ -10,13 +10,13 @@ class RssFeedBuilder
   include LogUtils::Logging
 
 
-  def self.build( rss_feed )
-    feed = self.new( rss_feed )
+  def self.build( rss_feed, raw )
+    feed = self.new( rss_feed, raw )
     feed.to_feed
   end
 
-  def initialize( rss_feed )
-    @feed = build_feed( rss_feed )
+  def initialize( rss_feed, raw )
+    @feed = build_feed( rss_feed, raw )
   end
 
   def to_feed
@@ -25,7 +25,7 @@ class RssFeedBuilder
 
 
 
-  def build_feed( rss_feed )
+  def build_feed( rss_feed, raw )
     feed = Feed.new
     feed.format = "rss #{rss_feed.rss_version}"
 
@@ -100,6 +100,14 @@ class RssFeedBuilder
       feed.items << build_item( rss_item )
     end
 
+    if defined?( Oga )
+      parsed_xml = Oga.parse_xml( raw )
+      xml_items = parsed_xml.xpath( '/rss/channel/item' )
+      xml_items.each_with_index do |xml_item, i|
+        feed.items[i] = add_meta_items( feed.items[i], xml_item )
+      end
+    end
+
     feed # return new feed
   end
 
@@ -221,6 +229,36 @@ class RssFeedBuilder
   end # method build_feed_item_from_rss
 
 
+  # Add additional elements, currently the media: namespace elements
+  def add_meta_items( feed_item, xml_item )
+    if xml_item.at_xpath('media:group') || xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content') || xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:description')
+      feed_item.attachments << Attachment.new unless feed_item.attachments.first
+
+      titleElement = xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content/media:title') || xml_item.at_xpath('media:group/media:title')
+      feed_item.attachments.first.title = titleElement.text if titleElement
+
+      contentElement = xml_item.at_xpath('media:content') || xml_item.at_xpath('media:group/media:content')
+      if contentElement
+        feed_item.attachments.first.url = contentElement.get('url')
+        feed_item.attachments.first.length = contentElement.get('duration')
+      end
+
+      thumbnailElement = xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:content/media:thumbnail') || xml_item.at_xpath('media:group/media:thumbnail')
+      if thumbnailElement
+        thumbnail = Thumbnail.new
+        thumbnail.url = thumbnailElement.get('url')
+        thumbnail.width = thumbnailElement.get('width')
+        thumbnail.height = thumbnailElement.get('height')
+        feed_item.attachments.first.thumbnail = thumbnail
+      end
+
+      descriptionElement = xml_item.at_xpath('media:description') || xml_item.at_xpath('media:content/media:description') || xml_item.at_xpath('media:group/media:description')
+      feed_item.attachments.first.description = descriptionElement.text if descriptionElement
+    end
+    feed_item
+  end # method add_meta_items
+
+
 
   def handle_date( el, name )
     ## change time to utc if present? why? why not?

data/lib/feedparser/parser.rb

@@ -110,9 +110,9 @@ class Parser
     logger.debug "  feed.class=#{feed_wild.class.name}"
 
     if feed_wild.is_a?( RSS::Atom::Feed )
-      feed = AtomFeedBuilder.build( feed_wild )
+      feed = AtomFeedBuilder.build( feed_wild, @text )
     else  # -- assume RSS::Rss::Feed
-      feed = RssFeedBuilder.build( feed_wild )
+      feed = RssFeedBuilder.build( feed_wild, @text )
     end
 
     logger.debug "== #{feed.format} / #{feed.title} =="

data/lib/feedparser/thumbnail.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+module FeedParser
+
+class Thumbnail
+
+  attr_accessor :url
+  
+  ## note: uri is an alias for url
+  alias :uri  :url       ## add atom alias for uri - why? why not?
+  alias :uri= :url=
+
+  def width?()   @width.nil? == false;  end
+  attr_accessor :width
+
+  def height?()  @height.nil? == false;  end
+  attr_accessor :height  # todo/check: use avatar_url ?? used by json feed -check if always a url
+
+end  # class Thumbnail
+
+end # module FeedParser

data/lib/feedparser/version.rb

@@ -3,8 +3,8 @@
 module FeedParser
 
   MAJOR = 2
-  MINOR = 1
-  PATCH = 2
+  MINOR = 2
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/media_rss_example.txt

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/" version="2.0">
+   <channel>
+      <title>Calm Meditation</title>
+      <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com</link>
+      <language>en-us</language>
+      <pubDate>Mon, 02 Apr 2018 16:19:56 -0700</pubDate>
+      <lastBuildDate>Mon, 02 Apr 2018 16:19:56 -0700</lastBuildDate>
+      <managingEditor>tomjoht@gmail.com (Tom Johnson)</managingEditor>
+      <description>Contains short videos capturing still scenes from nature with a music background, intended for calming or meditation purposes. When you're stressed out or upset, watch a few videos. As your mind focuses on the small details, let your worries and frustrations float away. The purpose is not to entertain or to distract, but to help calm, soothe, and surface your inner quiet. The videos contain scenes from the San Tomas Aquinas trail in Santa Clara, California.</description>
+      <image>
+         <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com</link>
+         <title>Calm Meditation</title>
+         <url>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/images/calmmeditationlogo_small.png</url>
+         <description>Contains short videos capturing still scenes from nature with a music background, intended for calming or meditation purposes. When you're stressed out or upset, watch a few videos. As your mind focuses on the small details, let your worries and frustrations float away. The purpose is not to entertain or to distract, but to help calm, soothe, and surface your inner quiet. The videos contain scenes from the San Tomas Aquinas trail in Santa Clara, California.</description>
+         <height>114</height>
+         <width>114</width>
+      </image>
+      <atom:link href="http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/feed.xml" rel="self" type="application/rss+xml" />
+      <item>
+         <title>Shade</title>
+         <pubDate>Mon, 23 Oct 2017 00:00:00 -0700</pubDate>
+         <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/shade/</link>
+         <description>Quiet the mind, and the soul will speak. - Ma Jaya Sati Bhagavati</description>
+         <guid isPermaLink="false">http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/shade/</guid>
+         <media:category>All</media:category>
+         <media:category>Trail</media:category>
+         <media:content url="http://d1nixf144dcz0j.cloudfront.net/shade.mp4" language="en-us" fileSize="37000000" duration="120.0" medium="video" isDefault="true">
+            <media:title type="plain">Shade</media:title>
+            <media:description type="html">Quiet the mind, and the soul will speak. - Ma Jaya Sati Bhagavati</media:description>
+            <media:thumbnail url="http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/images/thumbs/shade.jpg" />
+            <media:credit role="author" scheme="urn:ebu">Tom Johnson</media:credit>
+            <media:copyright url="https://creativecommons.org/licenses/by/4.0/" />
+         </media:content>
+      </item>
+      <item>
+         <title>Spectators</title>
+         <pubDate>Thu, 12 Oct 2017 00:00:00 -0700</pubDate>
+         <link>http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/spectators/</link>
+         <description>"Your worst enemy cannot harm you as much as your own thoughts, unguarded." – Buddha</description>
+         <guid isPermaLink="false">http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/spectators/</guid>
+         <media:category>All</media:category>
+         <media:category>Grass</media:category>
+         <media:content url="http://d1nixf144dcz0j.cloudfront.net/spectators.mp4" language="en-us" fileSize="19000000" duration="120.0" medium="video" isDefault="true">
+            <media:title type="plain">Spectators</media:title>
+            <media:description type="html">"Your worst enemy cannot harm you as much as your own thoughts, unguarded." – Buddha</media:description>
+            <media:thumbnail url="http://sample-firetv-web-app.s3-website-us-west-2.amazonaws.com/images/thumbs/spectators.jpg" />
+            <media:credit role="author" scheme="urn:ebu">Tom Johnson</media:credit>
+            <media:copyright url="https://creativecommons.org/licenses/by/4.0/" />
+         </media:content>
+      </item>
+   </channel>
+</rss>

data/test/test_attachments_live.rb

@@ -6,9 +6,17 @@
 
 require 'helper'
 
+
+###
+## note: needs to require oga gem (it's not required by default - it's a "soft" dependency)
+
+require 'oga'
+
+
+
 class TestAttachmentsLive < MiniTest::Test
 
-  def test_atom
+  def test_atom_enclose
     feed = fetch_and_parse_feed( 'http://www.lse.ac.uk/assets/richmedia/webFeeds/publicLecturesAndEvents_AtomAllMediaTypesLatest100.xml' )
 
     assert_equal 'audio/mpeg', feed.items.first.attachment.type
@@ -18,8 +26,37 @@ class TestAttachmentsLive < MiniTest::Test
     assert_equal true, feed.items.first.enclosure?
   end
 
+  def test_atom_media
+    feed = fetch_and_parse_feed( 'http://www.youtube.com/feeds/videos.xml?channel_id=UCZUT79WUUpZlZ-XMF7l4CFg' )
+    assert_equal true, feed.items.first.attachment?
+    assert feed.items.first.attachments.first.title
+    assert feed.items.first.attachments.first.url
+    assert feed.items.first.attachments.first.thumbnail
+    assert_instance_of FeedParser::Thumbnail, feed.items.first.attachments.first.thumbnail
+    assert feed.items.first.attachments.first.thumbnail.url
+    assert_equal 480, feed.items.first.attachments.first.thumbnail.width.to_i
+    assert_equal 360, feed.items.first.attachments.first.thumbnail.height.to_i
+    assert feed.items.first.attachments.first.description
+  end
+
+  def test_rss_media
+    # tests an example RSS file from https://creator.amazon.com/documentation/ac/mrss.html. Not that unlike the Atom example, it does
+    # does not put everything under media:group
+    testpath = File.join(File.expand_path(File.dirname(__FILE__)), 'media_rss_example.txt')
+    feed_rss = File.read( testpath )
+    feed = FeedParser::Parser.parse( feed_rss )
+    assert_equal true, feed.items.first.attachment?
+    assert feed.items.first.attachments.first.title
+    assert feed.items.first.attachments.first.url
+    assert feed.items.first.attachments.first.thumbnail
+    assert_instance_of FeedParser::Thumbnail, feed.items.first.attachments.first.thumbnail
+    assert feed.items.first.attachments.first.thumbnail.url
+    assert_nil feed.items.first.attachments.first.thumbnail.width
+    assert_nil feed.items.first.attachments.first.thumbnail.height
+    assert feed.items.first.attachments.first.description
+  end
 
-  def test_rss
+  def test_rss_enclosure
     feed = fetch_and_parse_feed( 'http://www.radiofreesatan.com/category/featured/feed/' )
 
     assert_equal 'audio/mpeg', feed.items.first.attachment.type

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedparser
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.2.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-30 00:00:00.000000000 Z
+date: 2020-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
@@ -92,8 +92,10 @@ files:
 - lib/feedparser/item.rb
 - lib/feedparser/parser.rb
 - lib/feedparser/tag.rb
+- lib/feedparser/thumbnail.rb
 - lib/feedparser/version.rb
 - test/helper.rb
+- test/media_rss_example.txt
 - test/test_atom_live.rb
 - test/test_attachments_live.rb
 - test/test_dates.rb