feedparser 2.1.2 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fdd14a8952605dcb40547f1675bc234fb16af21f
-  data.tar.gz: de37586ae452d80ebc2042ebfda9c29dabfa2aa0
+SHA256:
+  metadata.gz: daa2fcad3341d6e7ecc1beea5f05e5931c7763c863be715104a1317d491d774c
+  data.tar.gz: 95c673aa37b6bc8f81077155e5320f16723e7437356a39ac0082c4bf4661412e
 SHA512:
-  metadata.gz: b79665575a1596f1c644d48307e5660a52cf6ac1b0124bf6bc5df4ce66123cabada4dacebcf9240665dc2e0d5e2e52d14d764e9183bcf5bf25d9766385996b73
-  data.tar.gz: 8cb7a605620667f035a635826c29690835a4b38fa50e932d6b2fa6ac054d34f9c37a10585b5d9fcf0438836e3462081f85bb24a67ca2c1baa04fa4659cbd59eb
+  metadata.gz: aea5b0f826dd04602706cad85927e47656f3e2a91f930148460403f414ca11ffedc8736801b2ddb24decfa138bba580e70982e25612b54742aa06db47164836f
+  data.tar.gz: fecfa75ab226844282ee318f1760eed0abb9c7f7b76ce5404f3ef00b2bb85c055981252d30701a4cfa77347a308ce8cd91ed90c39a4e277ac29242c91cc4f299

data/CHANGELOG.md

@@ -1,3 +1,4 @@
+### 2.2.1
 ### 0.1.0 / 2013-09-19
 
 * Everything is new. First release.

data/Manifest.txt

@@ -14,8 +14,10 @@ lib/feedparser/generator.rb
 lib/feedparser/item.rb
 lib/feedparser/parser.rb
 lib/feedparser/tag.rb
+lib/feedparser/thumbnail.rb
 lib/feedparser/version.rb
 test/helper.rb
+test/media_rss_example.txt
 test/test_atom_live.rb
 test/test_attachments_live.rb
 test/test_dates.rb

data/Rakefile

@@ -8,10 +8,10 @@ Hoe.spec 'feedparser' do
   self.summary = 'feedparser - web feed parser and normalizer (RSS, Atom, JSON Feed, HTML h-entry, etc.)'
   self.description = summary
 
-  self.urls    = ['https://github.com/feedparser/feedparser']
+  self.urls    = { home: 'https://github.com/feedparser/feedparser' }
 
   self.author  = 'Gerald Bauer'
-  self.email   = 'wwwmake@googlegroups.com'
+  self.email   = 'gerald.bauer@gmail.com'
 
   # switch extension to .markdown for gihub formatting
   self.readme_file  = 'README.md'
@@ -20,6 +20,7 @@ Hoe.spec 'feedparser' do
   self.extra_deps = [
     ['logutils', '>=0.6.1'],
     ['textutils', '>=1.0.0'],
+    ## ['oga', '>=3.2.0'],   note: oga is a "soft" dependency
   ]
 
   ###  todo: add fetcher dep for testing (e.g. development only)

data/lib/feedparser/attachment.rb

@@ -1,17 +1,23 @@
-# encoding: utf-8
-
-module FeedParser
-
-class Attachment   ## also known as Enclosure
-
-  attr_accessor :url
-  ## note: uri is an alias for url
-  alias :uri  :url       ## add atom alias for uri - why? why not?
-  alias :uri= :url=
-
-  attr_accessor :length
-  attr_accessor :type
-
-end  # class Attachment
-
-end # module FeedParser
+# encoding: utf-8
+
+module FeedParser
+
+class Attachment   ## also known as Enclosure
+
+  attr_accessor :url
+  ## note: uri is an alias for url
+  alias :uri  :url       ## add atom alias for uri - why? why not?
+  alias :uri= :url=
+
+  attr_accessor :length
+  attr_accessor :type
+
+  # Elements from the media namespace attachment
+  attr_accessor :title
+  attr_accessor :thumbnail
+  attr_accessor :description
+  attr_accessor :community
+
+end  # class Attachment
+
+end # module FeedParser

data/lib/feedparser/author.rb

@@ -1,39 +1,39 @@
-# encoding: utf-8
-
-module FeedParser
-
-class Author
-
-  attr_accessor :name
-  attr_accessor :url
-  ## note: uri is an alias for url
-  alias :uri  :url       ## add atom alias for uri - why? why not?
-  alias :uri= :url=
-
-  def email?()   @email.nil? == false;  end
-  attr_accessor :email
-
-  def avatar?()  @avatar.nil? == false;  end
-  attr_accessor :avatar  # todo/check: use avatar_url ?? used by json feed -check if always a url
-
-
-  ## todo: add role - why? why not?
-  ##   e.g. add contributor (atom)
-  ##          or managingEditor (rss) or webMaster (rss) - why? why not??
-
-  attr_accessor :text    # note: holds "unparsed" text (content) line form dc:creator or rss:author
-  alias :line :text     # line|text  (add str??  too)
-
-  def to_s
-    ## note: to_s  - allows to use just author in templates
-    ##    will by default return name if present or as fallback "unparsed" text line
-     if @name    ## not blank
-       @name
-     else
-       @text
-     end
-  end
-
-end  # class Author
-
-end # module FeedParser
+# encoding: utf-8
+
+module FeedParser
+
+class Author
+
+  attr_accessor :name
+  attr_accessor :url
+  ## note: uri is an alias for url
+  alias :uri  :url       ## add atom alias for uri - why? why not?
+  alias :uri= :url=
+
+  def email?()   @email.nil? == false;  end
+  attr_accessor :email
+
+  def avatar?()  @avatar.nil? == false;  end
+  attr_accessor :avatar  # todo/check: use avatar_url ?? used by json feed -check if always a url
+
+
+  ## todo: add role - why? why not?
+  ##   e.g. add contributor (atom)
+  ##          or managingEditor (rss) or webMaster (rss) - why? why not??
+
+  attr_accessor :text    # note: holds "unparsed" text (content) line form dc:creator or rss:author
+  alias :line :text     # line|text  (add str??  too)
+
+  def to_s
+    ## note: to_s  - allows to use just author in templates
+    ##    will by default return name if present or as fallback "unparsed" text line
+     if @name    ## not blank
+       @name
+     else
+       @text
+     end
+  end
+
+end  # class Author
+
+end # module FeedParser

data/lib/feedparser/builder/atom.rb

@@ -7,13 +7,13 @@ class AtomFeedBuilder
   include LogUtils::Logging
 
 
-  def self.build( atom_feed )
-    feed = self.new( atom_feed )
+  def self.build( atom_feed, raw )
+    feed = self.new( atom_feed, raw )
     feed.to_feed
   end
 
-  def initialize( atom_feed )
-    @feed = build_feed( atom_feed )
+  def initialize( atom_feed, raw )
+    @feed = build_feed( atom_feed, raw )
   end
 
   def to_feed
@@ -22,7 +22,7 @@ class AtomFeedBuilder
 
 
 
-  def build_feed( atom_feed )    ## fix/todo: rename atom_feed to atom or wire or xml or in ???
+  def build_feed( atom_feed, raw )    ## fix/todo: rename atom_feed to atom or wire or xml or in ???
     feed = Feed.new
     feed.format = 'atom'
 
@@ -110,6 +110,16 @@ class AtomFeedBuilder
       feed.items << build_item( atom_item )
     end
 
+
+    if defined?( Oga )
+      # Use Oga as generic xml parser to access elements not adressed by the core RSS module like media:
+      parsed_xml = Oga.parse_xml( raw )
+      xml_items = parsed_xml.xpath( '/feed/entry' )
+      xml_items.each_with_index do |xml_item, i|
+          feed.items[i] = add_meta_items( feed.items[i], xml_item )
+      end
+    end
+
     feed # return new feed
   end # method build_feed_from_atom
 
@@ -221,6 +231,36 @@ class AtomFeedBuilder
   end # method build_item
 
 
+  # Add additional elements, currently the media: namespace elements
+  # Note: This tries to accomodate both the different ways to transport the data via the spec https://www.rssboard.org/media-rss/ and the practice by Youtube of grouping everything under media:group
+  def add_meta_items( feed_item, xml_item )
+    if xml_item.at_xpath('media:group') || xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content') || xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:description')
+      feed_item.attachments << Attachment.new unless feed_item.attachments.first
+
+      titleElement = xml_item.at_xpath('media:title') || xml_item.at_xpath('media:content/media:title') || xml_item.at_xpath('media:group/media:title')
+      feed_item.attachments.first.title = titleElement.text if titleElement
+
+      contentElement = xml_item.at_xpath('media:content') || xml_item.at_xpath('media:group/media:content')
+      if contentElement
+        feed_item.attachments.first.url = contentElement.get('url')
+        feed_item.attachments.first.length = contentElement.get('duration')
+      end
+
+      thumbnailElement = xml_item.at_xpath('media:thumbnail') || xml_item.at_xpath('media:content/media:thumbnail') || xml_item.at_xpath('media:group/media:thumbnail')
+      if thumbnailElement
+        thumbnail = Thumbnail.new
+        thumbnail.url = thumbnailElement.get('url')
+        thumbnail.width = thumbnailElement.get('width')
+        thumbnail.height = thumbnailElement.get('height')
+        feed_item.attachments.first.thumbnail = thumbnail
+      end
+
+      descriptionElement = xml_item.at_xpath('media:description') || xml_item.at_xpath('media:content/media:description') || xml_item.at_xpath('media:group/media:description')
+      feed_item.attachments.first.description = descriptionElement.text if descriptionElement
+    end
+    feed_item
+  end # method add_meta_items
+
 
   def handle_date( el, name )
     ## change time to utc if present? why? why not?

data/lib/feedparser/builder/json.rb

@@ -1,111 +1,111 @@
-# encoding: utf-8
-
-module FeedParser
-
-class JsonFeedBuilder
-
-  include LogUtils::Logging
-
-
-  def self.build( hash )
-    feed = self.new( hash )
-    feed.to_feed
-  end
-
-  def initialize( hash )
-    @feed = build_feed( hash )
-  end
-
-  def to_feed
-    @feed
-  end
-
-
-
-  def build_feed( h )
-    feed = Feed.new
-    feed.format = 'json'
-
-    feed.title    = h['title']
-    feed.url      = h['home_page_url']
-    feed.feed_url = h['feed_url']
-    feed.summary  = h['description']
-
-
-    if h['author']
-      feed.authors << build_author( h['author'] )
-    end
-
-
-    h['items'].each do |hash_item|
-      feed.items << build_item( hash_item )
-    end
-
-    feed # return new feed
-  end # method build_feed_from_json
-
-
-  def build_author( h )
-    author = Author.new
-
-    author.name     = h['name']
-    author.url      = h['url']
-    author.avatar   = h['avatar']
-
-    author
-  end
-
-
-
-  def build_item( h )
-    item = Item.new   # Item.new
-
-    item.guid         = h['id']
-    item.title        = h['title']
-    item.url          = h['url']
-    item.external_url = h['external_url']
-
-    ## convert date if present (from string to date type)
-    date_published_str = h['date_published']
-    if date_published_str
-      item.published_local  = DateTime.iso8601( date_published_str )
-      item.published        = item.published_local.utc
-    end
-
-    date_modified_str = h['date_modified']
-    if date_modified_str
-      item.updated_local  = DateTime.iso8601( date_modified_str )
-      item.updated        = item.updated_local.utc
-    end
-
-
-    item.content_html = h['content_html']
-    item.content_text = h['content_text']
-    item.summary      = h['summary']
-
-    if h['author']
-      item.authors << build_author( h['author'] )
-    end
-
-    if h['tags']
-      h['tags'].each do |json_tag|
-        item.tags << build_tag( json_tag )
-      end
-    end
-
-    item
-  end # method build_item
-
-
-  def build_tag( json_tag )
-    ## pp rss_cat
-    tag = Tag.new
-
-    tag.name = json_tag
-
-    tag
-  end  # build_tag
-
-
-end # JsonFeedBuilder
-end # FeedParser
+# encoding: utf-8
+
+module FeedParser
+
+class JsonFeedBuilder
+
+  include LogUtils::Logging
+
+
+  def self.build( hash )
+    feed = self.new( hash )
+    feed.to_feed
+  end
+
+  def initialize( hash )
+    @feed = build_feed( hash )
+  end
+
+  def to_feed
+    @feed
+  end
+
+
+
+  def build_feed( h )
+    feed = Feed.new
+    feed.format = 'json'
+
+    feed.title    = h['title']
+    feed.url      = h['home_page_url']
+    feed.feed_url = h['feed_url']
+    feed.summary  = h['description']
+
+
+    if h['author']
+      feed.authors << build_author( h['author'] )
+    end
+
+
+    h['items'].each do |hash_item|
+      feed.items << build_item( hash_item )
+    end
+
+    feed # return new feed
+  end # method build_feed_from_json
+
+
+  def build_author( h )
+    author = Author.new
+
+    author.name     = h['name']
+    author.url      = h['url']
+    author.avatar   = h['avatar']
+
+    author
+  end
+
+
+
+  def build_item( h )
+    item = Item.new   # Item.new
+
+    item.guid         = h['id']
+    item.title        = h['title']
+    item.url          = h['url']
+    item.external_url = h['external_url']
+
+    ## convert date if present (from string to date type)
+    date_published_str = h['date_published']
+    if date_published_str
+      item.published_local  = DateTime.iso8601( date_published_str )
+      item.published        = item.published_local.utc
+    end
+
+    date_modified_str = h['date_modified']
+    if date_modified_str
+      item.updated_local  = DateTime.iso8601( date_modified_str )
+      item.updated        = item.updated_local.utc
+    end
+
+
+    item.content_html = h['content_html']
+    item.content_text = h['content_text']
+    item.summary      = h['summary']
+
+    if h['author']
+      item.authors << build_author( h['author'] )
+    end
+
+    if h['tags']
+      h['tags'].each do |json_tag|
+        item.tags << build_tag( json_tag )
+      end
+    end
+
+    item
+  end # method build_item
+
+
+  def build_tag( json_tag )
+    ## pp rss_cat
+    tag = Tag.new
+
+    tag.name = json_tag
+
+    tag
+  end  # build_tag
+
+
+end # JsonFeedBuilder
+end # FeedParser