logstash-input-rss 2.0.5 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 097045b24b327a6d2fd4e4008490bfe9cf55d2ff
4
- data.tar.gz: 1ae11c5c5985551bb0cc473c7d918698aa50f18c
3
+ metadata.gz: c3c462cbab4cecbdfe73ffcc3113fd9bdcff69a0
4
+ data.tar.gz: b4d8690696754c3fc745589a39678a5ebd5e7d95
5
5
  SHA512:
6
- metadata.gz: 37acc14bb6f251a63f5f6de38b74ea17944e7016ab0752d54c9f42881bf62727a0251441e0d1b7ba3c5f282d150a93391980d7538aa360cadadfec558aec9dc4
7
- data.tar.gz: bab858afb5f867e67e15dacee3fb7aece3158f21907236b82dea0d06d0a2fa832acf2dbf2de021d69f86fe1d4fe45faffe8c38e5be743f68b6c727f5fb79b172
6
+ metadata.gz: eef2d87ebc599e7789c168182970083f40b52ece6c2918b62f3efe14c31bcb8a25e8ee9633187f29192ea525f51fa8328f6b4adadfdbd06106790ff3127ba3cf
7
+ data.tar.gz: 1e3810dccfc333b47f3c99f9318bc8a8b1cdc84dd43943ffe8daa7ca55beb522cf2de501fc7dfa888cc15aead182d67beab80f817cb8da2cb7fe7f0e2094d0f7
@@ -1,7 +1,17 @@
1
- # 2.0.5
1
+ ## 3.0.0
2
+ - Breaking: Updated plugin to use new Java Event APIs.
3
+ ## 2.0.6
4
+ - Fixes the case when there is an invalid feed, exceptions are now
5
+ catched and logged. Fixes https://github.com/logstash-plugins/logstash-input-rss/issues/1
6
+ - Also add test for normal, valid but empty and invalid feeds use
7
+ cases.
8
+
9
+ ## 2.0.5
2
10
  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
3
- # 2.0.4
11
+
12
+ ## 2.0.4
4
13
  - New dependency requirements for logstash-core for the 5.0 release
14
+
5
15
  ## 2.0.3
6
16
  - Fixed bug where `queue` variable was being used but not initialized in `handle_response`.
7
17
  Ref: https://github.com/logstash-plugins/logstash-input-rss/pull/14
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
1
+ Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
data/README.md CHANGED
@@ -1,7 +1,6 @@
1
1
  # Logstash Plugin
2
2
 
3
- [![Build
4
- Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-rss-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-rss-unit/)
3
+ [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-rss.svg)](https://travis-ci.org/logstash-plugins/logstash-input-rss)
5
4
 
6
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
7
6
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
56
55
  ```
57
56
  - Install plugin
58
57
  ```sh
58
+ # Logstash 2.3 and higher
59
+ bin/logstash-plugin install --no-verify
60
+
61
+ # Prior to Logstash 2.3
59
62
  bin/plugin install --no-verify
63
+
60
64
  ```
61
65
  - Run Logstash with your plugin
62
66
  ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
74
78
  ```
75
79
  - Install the plugin from the Logstash home
76
80
  ```sh
77
- bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
81
+ # Logstash 2.3 and higher
82
+ bin/logstash-plugin install --no-verify
83
+
84
+ # Prior to Logstash 2.3
85
+ bin/plugin install --no-verify
86
+
78
87
  ```
79
88
  - Start Logstash and proceed to test the plugin
80
89
 
@@ -61,46 +61,62 @@ class LogStash::Inputs::Rss < LogStash::Inputs::Base
61
61
 
62
62
  def handle_response(response, queue)
63
63
  body = response.body
64
- # @logger.debug("Body", :body => body)
65
- # Parse the RSS feed
66
- feed = RSS::Parser.parse(body)
67
- feed.items.each do |item|
68
- # Put each item into an event
69
- @logger.debug("Item", :item => item.author)
70
- case feed.feed_type
64
+ begin
65
+ feed = RSS::Parser.parse(body)
66
+ feed.items.each do |item|
67
+ # Put each item into an event
68
+ @logger.debug("Item", :item => item.author)
69
+ case feed.feed_type
71
70
  when 'rss'
72
- @codec.decode(item.description) do |event|
73
- event["Feed"] = @url
74
- event["published"] = item.pubDate
75
- event["title"] = item.title
76
- event["link"] = item.link
77
- event["author"] = item.author
78
- decorate(event)
79
- queue << event
80
- end
71
+ handle_rss_response(queue, item)
81
72
  when 'atom'
82
- if ! item.content.nil?
83
- content = item.content.content
84
- else
85
- content = item.summary.content
86
- end
87
- @codec.decode(content) do |event|
88
- event["Feed"] = @url
89
- event["updated"] = item.updated.content
90
- event["title"] = item.title.content
91
- event["link"] = item.link.href
92
- event["author"] = item.author.name.content
93
- unless item.published.nil?
94
- event["published"] = item.published.content
95
- end
96
- decorate(event)
97
- queue << event
98
- end
73
+ handle_atom_response(queue, item)
74
+ end
99
75
  end
76
+ rescue RSS::MissingTagError => e
77
+ @logger.error("Invalid RSS feed", :exception => e)
78
+ rescue => e
79
+ @logger.error("Uknown error while parsing the feed", :url => url, :exception => e)
100
80
  end
101
81
  end
102
82
 
103
83
  def stop
104
84
  Stud.stop!(@run_thread) if @run_thread
105
85
  end
86
+
87
+ private
88
+
89
+ def handle_atom_response(queue, item)
90
+ if ! item.content.nil?
91
+ content = item.content.content
92
+ else
93
+ content = item.summary.content
94
+ end
95
+ @codec.decode(content) do |event|
96
+ event.set("Feed", @url)
97
+ event.set("updated", item.updated.content)
98
+ event.set("title", item.title.content)
99
+ event.set("link", item.link.href)
100
+ ##
101
+ # Author is actually a recommended field, not not a mandatory
102
+ # one, see https://validator.w3.org/feed/docs/atom.html for details.
103
+ ##
104
+ event.set("author", item.author.name.content) if !item.author.nil?
105
+ event.set("published", item.published.content) if !item.published.nil?
106
+
107
+ decorate(event)
108
+ queue << event
109
+ end
110
+ end
111
+ def handle_rss_response(queue, item)
112
+ @codec.decode(item.description) do |event|
113
+ event.set("Feed", @url)
114
+ event.set("published", item.pubDate)
115
+ event.set("title", item.title)
116
+ event.set("link", item.link)
117
+ event.set("author", item.author)
118
+ decorate(event)
119
+ queue << event
120
+ end
121
+ end
106
122
  end # class LogStash::Inputs::Exec
@@ -1,10 +1,10 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-rss'
4
- s.version = '2.0.5'
4
+ s.version = '3.0.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Poll an RSS/Atom feed."
7
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ["Elastic"]
9
9
  s.email = 'jason.kendall@elastic.co'
10
10
  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
20
20
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
21
21
 
22
22
  # Gem dependencies
23
- s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
23
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
24
24
  s.add_runtime_dependency 'logstash-codec-plain'
25
25
  s.add_runtime_dependency 'addressable'
26
26
  s.add_runtime_dependency 'faraday'
@@ -0,0 +1,3 @@
1
+ <?xml version="1.0" encoding="utf-8"?>
2
+ <feed xmlns="http://www.w3.org/2005/Atom">
3
+ </feed>
@@ -0,0 +1,19 @@
1
+ <?xml version="1.0" encoding="utf-8"?>
2
+ <feed xmlns="http://www.w3.org/2005/Atom">
3
+ <title>Example Feed</title>
4
+ <link href="http://example.org/"/>
5
+ <updated>2003-12-13T18:30:02Z</updated>
6
+ <author>
7
+ <name>John Doe</name>
8
+ </author>
9
+ <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
10
+
11
+ <entry>
12
+ <title>Atom-Powered Robots Run Amok</title>
13
+ <link href="http://example.org/2003/12/13/atom03"/>
14
+ <id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
15
+ <updated>2003-12-13T18:30:02Z</updated>
16
+ <summary>Some text.</summary>
17
+ </entry>
18
+
19
+ </feed>
@@ -0,0 +1,10 @@
1
+ <?xml version="1.0" encoding="utf-8"?>
2
+ <feed xmlns="http://www.w3.org/2005/Atom">
3
+ <title>Example Feed</title>
4
+ <link href="http://example.org/"/>
5
+ <updated>2003-12-13T18:30:02Z</updated>
6
+ <author>
7
+ <name>John Doe</name>
8
+ </author>
9
+ <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
10
+ </feed>
@@ -0,0 +1,5 @@
1
+ <?xml version="1.0" encoding="windows-1252"?>
2
+ <rss version="2.0">
3
+ <channel>
4
+ </channel>
5
+ </rss>
@@ -0,0 +1,52 @@
1
+ <?xml version="1.0" encoding="windows-1252"?>
2
+ <rss version="2.0">
3
+ <channel>
4
+ <title>Sample Feed - Favorite RSS Related Software &amp; Resources</title>
5
+ <description>Take a look at some of FeedForAll&apos;s favorite software and resources for learning more about RSS.</description>
6
+ <link>http://www.feedforall.com</link>
7
+ <category domain="www.dmoz.com">Computers/Software/Internet/Site Management/Content Management</category>
8
+ <copyright>Copyright 2004 NotePage, Inc.</copyright>
9
+ <docs>http://blogs.law.harvard.edu/tech/rss</docs>
10
+ <language>en-us</language>
11
+ <lastBuildDate>Mon, 1 Nov 2004 13:17:17 -0500</lastBuildDate>
12
+ <managingEditor>marketing@feedforall.com</managingEditor>
13
+ <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
14
+ <webMaster>webmaster@feedforall.com</webMaster>
15
+ <generator>FeedForAll Beta1 (0.0.1.8)</generator>
16
+ <image>
17
+ <url>http://www.feedforall.com/feedforall-temp.gif</url>
18
+ <title>FeedForAll Sample Feed</title>
19
+ <link>http://www.feedforall.com/industry-solutions.htm</link>
20
+ <description>FeedForAll Sample Feed</description>
21
+ <width>144</width>
22
+ <height>117</height>
23
+ </image>
24
+ <item>
25
+ <title>RSS Resources</title>
26
+ <description>Be sure to take a look at some of our favorite RSS Resources&lt;br&gt;
27
+ &lt;a href=&quot;http://www.rss-specifications.com&quot;&gt;RSS Specifications&lt;/a&gt;&lt;br&gt;
28
+ &lt;a href=&quot;http://www.blog-connection.com&quot;&gt;Blog Connection&lt;/a&gt;&lt;br&gt;
29
+ &lt;br&gt;</description>
30
+ <link>http://www.feedforall.com</link>
31
+ <pubDate>Tue, 26 Oct 2004 14:01:01 -0500</pubDate>
32
+ </item>
33
+ <item>
34
+ <title>Recommended Desktop Feed Reader Software</title>
35
+ <description>&lt;b&gt;FeedDemon&lt;/b&gt; enables you to quickly read and gather information from hundreds of web sites - without having to visit them. Don&apos;t waste any more time checking your favorite web sites for updates. Instead, use FeedDemon and make them come to you. &lt;br&gt;
36
+ More &lt;a href=&quot;http://store.esellerate.net/a.asp?c=1_SKU5139890208_AFL403073819&quot;&gt;FeedDemon Information&lt;/a&gt;</description>
37
+ <link>http://www.feedforall.com/feedforall-partners.htm</link>
38
+ <pubDate>Tue, 26 Oct 2004 14:03:25 -0500</pubDate>
39
+ </item>
40
+ <item>
41
+ <title>Recommended Web Based Feed Reader Software</title>
42
+ <description>&lt;b&gt;FeedScout&lt;/b&gt; enables you to view RSS/ATOM/RDF feeds from different sites directly in Internet Explorer. You can even set your Home Page to show favorite feeds. Feed Scout is a plug-in for Internet Explorer, so you won&apos;t have to learn anything except for how to press 2 new buttons on Internet Explorer toolbar. &lt;br&gt;
43
+ More &lt;a href=&quot;http://www.bytescout.com/feedscout.html&quot;&gt;Information on FeedScout&lt;/a&gt;&lt;br&gt;
44
+ &lt;br&gt;
45
+ &lt;br&gt;
46
+ &lt;b&gt;SurfPack&lt;/b&gt; can feature search tools, horoscopes, current weather conditions, LiveJournal diaries, humor, web modules and other dynamically updated content. &lt;br&gt;
47
+ More &lt;a href=&quot;http://www.surfpack.com/&quot;&gt;Information on SurfPack&lt;/a&gt;&lt;br&gt;</description>
48
+ <link>http://www.feedforall.com/feedforall-partners.htm</link>
49
+ <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
50
+ </item>
51
+ </channel>
52
+ </rss>
@@ -0,0 +1,8 @@
1
+ <?xml version="1.0" encoding="windows-1252"?>
2
+ <rss version="2.0">
3
+ <channel>
4
+ <title>Sample Feed - Favorite RSS Related Software &amp; Resources</title>
5
+ <description>Take a look at some of FeedForAll&apos;s favorite software and resources for learning more about RSS.</description>
6
+ <link>http://www.feedforall.com</link>
7
+ </channel>
8
+ </rss>
@@ -1,6 +1,7 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/inputs/rss"
4
+ require 'ostruct'
4
5
 
5
6
  describe LogStash::Inputs::Rss do
6
7
  describe "stopping" do
@@ -11,4 +12,96 @@ describe LogStash::Inputs::Rss do
11
12
  end
12
13
  it_behaves_like "an interruptible input plugin"
13
14
  end
14
- end
15
+
16
+ shared_examples "fetching data" do |type|
17
+ let(:config) do
18
+ {
19
+ "url" => "http://www.example.com/foo.rss",
20
+ "interval" => 10
21
+ }
22
+ end
23
+
24
+ let(:sample) do
25
+ body = File.read(File.join(fixtures_source, "sample-feed.xml"))
26
+ OpenStruct.new(:body => body)
27
+ end
28
+
29
+ before(:each) do
30
+ allow(Faraday).to receive(:get).with(config["url"]).and_return(sample)
31
+ end
32
+
33
+ context "when the feed is valid" do
34
+ let(:data) do
35
+ plugin = described_class.new(config)
36
+ plugin_input(plugin) do |queue|
37
+ sleep 0.1 while queue.empty?
38
+ events = []
39
+ queue.size.times { |i| events << queue.pop }
40
+ events
41
+ end
42
+ end
43
+
44
+ it "fetchs all items" do
45
+ expect(data.count).to be > 0
46
+ end
47
+ end
48
+
49
+ context "when the feed is invalid" do
50
+
51
+ let(:sample) do
52
+ body = File.read(File.join(fixtures_source, "invalid-feed.xml"))
53
+ OpenStruct.new(:body => body)
54
+ end
55
+
56
+ let(:plugin) { described_class.new(config) }
57
+
58
+ it "fetchs no items and causes no errors" do
59
+ events = []
60
+ expect {
61
+ plugin_input(plugin) do |queue|
62
+ sleep 1
63
+ events = []
64
+ queue.size.times { |i| events << queue.pop }
65
+ events
66
+ end
67
+ }.not_to raise_error
68
+ expect(events.count).to be == 0
69
+ end
70
+ end
71
+
72
+ context "when the feed is valid, but has zero items" do
73
+
74
+ let(:sample) do
75
+ body = File.read(File.join(fixtures_source, "zero-items-feed.xml"))
76
+ OpenStruct.new(:body => body)
77
+ end
78
+
79
+ let(:plugin) { described_class.new(config) }
80
+
81
+ it "fetchs no items and causes no errors" do
82
+ events = []
83
+ expect {
84
+ plugin_input(plugin) do |queue|
85
+ sleep 1
86
+ events = []
87
+ queue.size.times { |i| events << queue.pop }
88
+ events
89
+ end
90
+ }.not_to raise_error
91
+ expect(events.count).to be == 0
92
+ end
93
+ end
94
+
95
+ end
96
+
97
+ describe "rss feed" do
98
+ let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "rss") }
99
+ it_behaves_like "fetching data"
100
+ end
101
+
102
+ describe "atom feed" do
103
+ let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "atom") }
104
+ it_behaves_like "fetching data"
105
+ end
106
+
107
+ end
metadata CHANGED
@@ -1,21 +1,21 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.5
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-24 00:00:00.000000000 Z
11
+ date: 2016-07-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
16
  - - "~>"
17
17
  - !ruby/object:Gem::Version
18
- version: '1.0'
18
+ version: '2.0'
19
19
  name: logstash-core-plugin-api
20
20
  prerelease: false
21
21
  type: :runtime
@@ -23,7 +23,7 @@ dependencies:
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.0'
26
+ version: '2.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
@@ -94,7 +94,7 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
- description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
97
+ description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
98
98
  email: jason.kendall@elastic.co
99
99
  executables: []
100
100
  extensions: []
@@ -108,6 +108,12 @@ files:
108
108
  - README.md
109
109
  - lib/logstash/inputs/rss.rb
110
110
  - logstash-input-rss.gemspec
111
+ - spec/fixtures/atom/invalid-feed.xml
112
+ - spec/fixtures/atom/sample-feed.xml
113
+ - spec/fixtures/atom/zero-items-feed.xml
114
+ - spec/fixtures/rss/invalid-feed.xml
115
+ - spec/fixtures/rss/sample-feed.xml
116
+ - spec/fixtures/rss/zero-items-feed.xml
111
117
  - spec/inputs/rss_spec.rb
112
118
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
113
119
  licenses:
@@ -136,4 +142,10 @@ signing_key:
136
142
  specification_version: 4
137
143
  summary: Poll an RSS/Atom feed.
138
144
  test_files:
145
+ - spec/fixtures/atom/invalid-feed.xml
146
+ - spec/fixtures/atom/sample-feed.xml
147
+ - spec/fixtures/atom/zero-items-feed.xml
148
+ - spec/fixtures/rss/invalid-feed.xml
149
+ - spec/fixtures/rss/sample-feed.xml
150
+ - spec/fixtures/rss/zero-items-feed.xml
139
151
  - spec/inputs/rss_spec.rb