logstash-input-rss 2.0.5 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 097045b24b327a6d2fd4e4008490bfe9cf55d2ff
-  data.tar.gz: 1ae11c5c5985551bb0cc473c7d918698aa50f18c
+  metadata.gz: c3c462cbab4cecbdfe73ffcc3113fd9bdcff69a0
+  data.tar.gz: b4d8690696754c3fc745589a39678a5ebd5e7d95
 SHA512:
-  metadata.gz: 37acc14bb6f251a63f5f6de38b74ea17944e7016ab0752d54c9f42881bf62727a0251441e0d1b7ba3c5f282d150a93391980d7538aa360cadadfec558aec9dc4
-  data.tar.gz: bab858afb5f867e67e15dacee3fb7aece3158f21907236b82dea0d06d0a2fa832acf2dbf2de021d69f86fe1d4fe45faffe8c38e5be743f68b6c727f5fb79b172
+  metadata.gz: eef2d87ebc599e7789c168182970083f40b52ece6c2918b62f3efe14c31bcb8a25e8ee9633187f29192ea525f51fa8328f6b4adadfdbd06106790ff3127ba3cf
+  data.tar.gz: 1e3810dccfc333b47f3c99f9318bc8a8b1cdc84dd43943ffe8daa7ca55beb522cf2de501fc7dfa888cc15aead182d67beab80f817cb8da2cb7fe7f0e2094d0f7

data/CHANGELOG.md

@@ -1,7 +1,17 @@
-# 2.0.5
+## 3.0.0
+  - Breaking: Updated plugin to use new Java Event APIs.
+## 2.0.6
+  - Fixes the case when there is an invalid feed, exceptions are now
+    catched and logged. Fixes https://github.com/logstash-plugins/logstash-input-rss/issues/1
+  - Also add test for normal, valid but empty and invalid feeds use
+    cases.
+
+## 2.0.5
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.4
+
+## 2.0.4
   - New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.0.3
  - Fixed bug where `queue` variable was being used but not initialized in `handle_response`.
    Ref: https://github.com/logstash-plugins/logstash-input-rss/pull/14

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-rss-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-rss-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-rss.svg)](https://travis-ci.org/logstash-plugins/logstash-input-rss)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/inputs/rss.rb

@@ -61,46 +61,62 @@ class LogStash::Inputs::Rss < LogStash::Inputs::Base
 
   def handle_response(response, queue)
     body = response.body
-    # @logger.debug("Body", :body => body)
-    # Parse the RSS feed
-    feed = RSS::Parser.parse(body)
-    feed.items.each do |item|
-      # Put each item into an event
-      @logger.debug("Item", :item => item.author)
-      case feed.feed_type
+    begin
+      feed = RSS::Parser.parse(body)
+      feed.items.each do |item|
+        # Put each item into an event
+        @logger.debug("Item", :item => item.author)
+        case feed.feed_type
         when 'rss'
-          @codec.decode(item.description) do |event|
-            event["Feed"] = @url
-            event["published"] = item.pubDate
-            event["title"] = item.title
-            event["link"] = item.link
-            event["author"] = item.author
-            decorate(event)
-            queue << event
-          end
+          handle_rss_response(queue, item)
         when 'atom'
-          if ! item.content.nil?
-            content = item.content.content
-          else
-            content = item.summary.content
-          end
-          @codec.decode(content) do |event|
-            event["Feed"] = @url
-            event["updated"] = item.updated.content
-            event["title"] = item.title.content
-            event["link"] = item.link.href
-            event["author"] = item.author.name.content
-            unless item.published.nil?
-              event["published"] = item.published.content
-            end
-            decorate(event)
-            queue << event
-          end
+          handle_atom_response(queue, item)
+        end
       end
+    rescue RSS::MissingTagError => e
+      @logger.error("Invalid RSS feed", :exception => e)
+    rescue => e
+      @logger.error("Uknown error while parsing the feed", :url => url, :exception => e)
     end
   end
 
   def stop
     Stud.stop!(@run_thread) if @run_thread
   end
+
+  private
+
+  def handle_atom_response(queue, item)
+    if ! item.content.nil?
+      content = item.content.content
+    else
+      content = item.summary.content
+    end
+    @codec.decode(content) do |event|
+      event.set("Feed", @url)
+      event.set("updated", item.updated.content)
+      event.set("title", item.title.content)
+      event.set("link", item.link.href)
+      ##
+      # Author is actually a recommended field, not not a mandatory 
+      # one, see https://validator.w3.org/feed/docs/atom.html for details.
+      ##
+      event.set("author", item.author.name.content) if !item.author.nil?
+      event.set("published", item.published.content) if !item.published.nil?
+
+      decorate(event)
+      queue << event
+    end
+  end
+  def handle_rss_response(queue, item)
+    @codec.decode(item.description) do |event|
+      event.set("Feed",  @url)
+      event.set("published", item.pubDate)
+      event.set("title", item.title)
+      event.set("link", item.link)
+      event.set("author", item.author)
+      decorate(event)
+      queue << event
+    end
+  end
 end # class LogStash::Inputs::Exec

data/logstash-input-rss.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-rss'
-  s.version         = '2.0.5'
+  s.version         = '3.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Poll an RSS/Atom feed."
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'jason.kendall@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'addressable'
   s.add_runtime_dependency 'faraday'

data/spec/fixtures/atom/invalid-feed.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+</feed>

data/spec/fixtures/atom/sample-feed.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>Example Feed</title>
+  <link href="http://example.org/"/>
+  <updated>2003-12-13T18:30:02Z</updated>
+  <author>
+    <name>John Doe</name>
+  </author>
+  <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
+
+  <entry>
+    <title>Atom-Powered Robots Run Amok</title>
+    <link href="http://example.org/2003/12/13/atom03"/>
+    <id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
+    <updated>2003-12-13T18:30:02Z</updated>
+    <summary>Some text.</summary>
+  </entry>
+
+</feed>

data/spec/fixtures/atom/zero-items-feed.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>Example Feed</title>
+  <link href="http://example.org/"/>
+  <updated>2003-12-13T18:30:02Z</updated>
+  <author>
+    <name>John Doe</name>
+  </author>
+  <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
+</feed>

data/spec/fixtures/rss/invalid-feed.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="windows-1252"?>
+<rss version="2.0">
+  <channel>
+   </channel>
+</rss>

data/spec/fixtures/rss/sample-feed.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="windows-1252"?>
+<rss version="2.0">
+  <channel>
+    <title>Sample Feed - Favorite RSS Related Software &amp; Resources</title>
+    <description>Take a look at some of FeedForAll&apos;s favorite software and resources for learning more about RSS.</description>
+    <link>http://www.feedforall.com</link>
+    <category domain="www.dmoz.com">Computers/Software/Internet/Site Management/Content Management</category>
+    <copyright>Copyright 2004 NotePage, Inc.</copyright>
+    <docs>http://blogs.law.harvard.edu/tech/rss</docs>
+    <language>en-us</language>
+    <lastBuildDate>Mon, 1 Nov 2004 13:17:17 -0500</lastBuildDate>
+    <managingEditor>marketing@feedforall.com</managingEditor>
+    <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
+    <webMaster>webmaster@feedforall.com</webMaster>
+    <generator>FeedForAll Beta1 (0.0.1.8)</generator>
+    <image>
+      <url>http://www.feedforall.com/feedforall-temp.gif</url>
+      <title>FeedForAll Sample Feed</title>
+      <link>http://www.feedforall.com/industry-solutions.htm</link>
+      <description>FeedForAll Sample Feed</description>
+      <width>144</width>
+      <height>117</height>
+    </image>
+    <item>
+      <title>RSS Resources</title>
+      <description>Be sure to take a look at some of our favorite RSS Resources&lt;br&gt;
+&lt;a href=&quot;http://www.rss-specifications.com&quot;&gt;RSS Specifications&lt;/a&gt;&lt;br&gt;
+&lt;a href=&quot;http://www.blog-connection.com&quot;&gt;Blog Connection&lt;/a&gt;&lt;br&gt;
+&lt;br&gt;</description>
+      <link>http://www.feedforall.com</link>
+      <pubDate>Tue, 26 Oct 2004 14:01:01 -0500</pubDate>
+    </item>
+    <item>
+      <title>Recommended Desktop Feed Reader Software</title>
+      <description>&lt;b&gt;FeedDemon&lt;/b&gt; enables you to quickly read and gather information from hundreds of web sites - without having to visit them. Don&apos;t waste any more time checking your favorite web sites for updates. Instead, use FeedDemon and make them come to you. &lt;br&gt;
+More &lt;a href=&quot;http://store.esellerate.net/a.asp?c=1_SKU5139890208_AFL403073819&quot;&gt;FeedDemon Information&lt;/a&gt;</description>
+      <link>http://www.feedforall.com/feedforall-partners.htm</link>
+      <pubDate>Tue, 26 Oct 2004 14:03:25 -0500</pubDate>
+    </item>
+    <item>
+      <title>Recommended Web Based Feed Reader Software</title>
+      <description>&lt;b&gt;FeedScout&lt;/b&gt; enables you to view RSS/ATOM/RDF feeds from different sites directly in Internet Explorer. You can even set your Home Page to show favorite feeds. Feed Scout is a plug-in for Internet Explorer, so you won&apos;t have to learn anything except for how to press 2 new buttons on Internet Explorer toolbar. &lt;br&gt;
+More &lt;a href=&quot;http://www.bytescout.com/feedscout.html&quot;&gt;Information on FeedScout&lt;/a&gt;&lt;br&gt;
+&lt;br&gt;
+&lt;br&gt;
+&lt;b&gt;SurfPack&lt;/b&gt; can feature search tools, horoscopes, current weather conditions, LiveJournal diaries, humor, web modules and other dynamically updated content. &lt;br&gt;
+More &lt;a href=&quot;http://www.surfpack.com/&quot;&gt;Information on SurfPack&lt;/a&gt;&lt;br&gt;</description>
+      <link>http://www.feedforall.com/feedforall-partners.htm</link>
+      <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
+    </item>
+  </channel>
+</rss>

data/spec/fixtures/rss/zero-items-feed.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="windows-1252"?>
+<rss version="2.0">
+  <channel>
+    <title>Sample Feed - Favorite RSS Related Software &amp; Resources</title>
+    <description>Take a look at some of FeedForAll&apos;s favorite software and resources for learning more about RSS.</description>
+    <link>http://www.feedforall.com</link>
+   </channel>
+</rss>

data/spec/inputs/rss_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/rss"
+require 'ostruct'
 
 describe LogStash::Inputs::Rss do
   describe "stopping" do
@@ -11,4 +12,96 @@ describe LogStash::Inputs::Rss do
     end
     it_behaves_like "an interruptible input plugin"
   end
-end
+
+  shared_examples "fetching data" do |type|
+    let(:config) do
+      {
+        "url" => "http://www.example.com/foo.rss",
+        "interval" => 10
+      }
+    end
+
+    let(:sample) do
+      body = File.read(File.join(fixtures_source, "sample-feed.xml"))
+      OpenStruct.new(:body => body)
+    end
+
+    before(:each) do
+      allow(Faraday).to receive(:get).with(config["url"]).and_return(sample)
+    end
+
+    context "when the feed is valid" do
+      let(:data) do
+        plugin = described_class.new(config)
+        plugin_input(plugin) do |queue|
+          sleep 0.1 while queue.empty?
+          events = []
+          queue.size.times { |i| events << queue.pop }
+          events
+        end
+      end
+
+      it "fetchs all items" do
+        expect(data.count).to be > 0
+      end
+    end
+
+    context "when the feed is invalid" do
+
+      let(:sample) do
+        body = File.read(File.join(fixtures_source, "invalid-feed.xml"))
+        OpenStruct.new(:body => body)
+      end
+
+      let(:plugin) { described_class.new(config) }
+
+      it "fetchs no items and causes no errors" do
+        events = []
+        expect {
+          plugin_input(plugin) do |queue|
+            sleep 1
+            events = []
+            queue.size.times { |i| events << queue.pop }
+            events
+          end
+        }.not_to raise_error
+        expect(events.count).to be == 0
+      end
+    end
+
+    context "when the feed is valid, but has zero items" do
+
+      let(:sample) do
+        body = File.read(File.join(fixtures_source, "zero-items-feed.xml"))
+        OpenStruct.new(:body => body)
+      end
+
+      let(:plugin) { described_class.new(config) }
+
+      it "fetchs no items and causes no errors" do
+        events = []
+        expect {
+          plugin_input(plugin) do |queue|
+            sleep 1
+            events = []
+            queue.size.times { |i| events << queue.pop }
+            events
+          end
+        }.not_to raise_error
+        expect(events.count).to be == 0
+      end
+    end
+
+  end
+
+  describe "rss feed" do
+    let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "rss")  }
+    it_behaves_like "fetching data"
+  end
+
+  describe "atom feed" do
+    let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "atom")  }
+    it_behaves_like "fetching data"
+  end
+
+end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-rss
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 3.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -94,7 +94,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: jason.kendall@elastic.co
 executables: []
 extensions: []
@@ -108,6 +108,12 @@ files:
 - README.md
 - lib/logstash/inputs/rss.rb
 - logstash-input-rss.gemspec
+- spec/fixtures/atom/invalid-feed.xml
+- spec/fixtures/atom/sample-feed.xml
+- spec/fixtures/atom/zero-items-feed.xml
+- spec/fixtures/rss/invalid-feed.xml
+- spec/fixtures/rss/sample-feed.xml
+- spec/fixtures/rss/zero-items-feed.xml
 - spec/inputs/rss_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -136,4 +142,10 @@ signing_key:
 specification_version: 4
 summary: Poll an RSS/Atom feed.
 test_files:
+- spec/fixtures/atom/invalid-feed.xml
+- spec/fixtures/atom/sample-feed.xml
+- spec/fixtures/atom/zero-items-feed.xml
+- spec/fixtures/rss/invalid-feed.xml
+- spec/fixtures/rss/sample-feed.xml
+- spec/fixtures/rss/zero-items-feed.xml
 - spec/inputs/rss_spec.rb