logstash-input-rss 2.0.5 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 097045b24b327a6d2fd4e4008490bfe9cf55d2ff
4
- data.tar.gz: 1ae11c5c5985551bb0cc473c7d918698aa50f18c
3
+ metadata.gz: c3c462cbab4cecbdfe73ffcc3113fd9bdcff69a0
4
+ data.tar.gz: b4d8690696754c3fc745589a39678a5ebd5e7d95
5
5
  SHA512:
6
- metadata.gz: 37acc14bb6f251a63f5f6de38b74ea17944e7016ab0752d54c9f42881bf62727a0251441e0d1b7ba3c5f282d150a93391980d7538aa360cadadfec558aec9dc4
7
- data.tar.gz: bab858afb5f867e67e15dacee3fb7aece3158f21907236b82dea0d06d0a2fa832acf2dbf2de021d69f86fe1d4fe45faffe8c38e5be743f68b6c727f5fb79b172
6
+ metadata.gz: eef2d87ebc599e7789c168182970083f40b52ece6c2918b62f3efe14c31bcb8a25e8ee9633187f29192ea525f51fa8328f6b4adadfdbd06106790ff3127ba3cf
7
+ data.tar.gz: 1e3810dccfc333b47f3c99f9318bc8a8b1cdc84dd43943ffe8daa7ca55beb522cf2de501fc7dfa888cc15aead182d67beab80f817cb8da2cb7fe7f0e2094d0f7
@@ -1,7 +1,17 @@
1
- # 2.0.5
1
+ ## 3.0.0
2
+ - Breaking: Updated plugin to use new Java Event APIs.
3
+ ## 2.0.6
4
+ - Fixes the case when there is an invalid feed, exceptions are now
5
+ catched and logged. Fixes https://github.com/logstash-plugins/logstash-input-rss/issues/1
6
+ - Also add test for normal, valid but empty and invalid feeds use
7
+ cases.
8
+
9
+ ## 2.0.5
2
10
  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
3
- # 2.0.4
11
+
12
+ ## 2.0.4
4
13
  - New dependency requirements for logstash-core for the 5.0 release
14
+
5
15
  ## 2.0.3
6
16
  - Fixed bug where `queue` variable was being used but not initialized in `handle_response`.
7
17
  Ref: https://github.com/logstash-plugins/logstash-input-rss/pull/14
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
1
+ Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
data/README.md CHANGED
@@ -1,7 +1,6 @@
1
1
  # Logstash Plugin
2
2
 
3
- [![Build
4
- Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-rss-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-rss-unit/)
3
+ [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-rss.svg)](https://travis-ci.org/logstash-plugins/logstash-input-rss)
5
4
 
6
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
7
6
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
56
55
  ```
57
56
  - Install plugin
58
57
  ```sh
58
+ # Logstash 2.3 and higher
59
+ bin/logstash-plugin install --no-verify
60
+
61
+ # Prior to Logstash 2.3
59
62
  bin/plugin install --no-verify
63
+
60
64
  ```
61
65
  - Run Logstash with your plugin
62
66
  ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
74
78
  ```
75
79
  - Install the plugin from the Logstash home
76
80
  ```sh
77
- bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
81
+ # Logstash 2.3 and higher
82
+ bin/logstash-plugin install --no-verify
83
+
84
+ # Prior to Logstash 2.3
85
+ bin/plugin install --no-verify
86
+
78
87
  ```
79
88
  - Start Logstash and proceed to test the plugin
80
89
 
@@ -61,46 +61,62 @@ class LogStash::Inputs::Rss < LogStash::Inputs::Base
61
61
 
62
62
  def handle_response(response, queue)
63
63
  body = response.body
64
- # @logger.debug("Body", :body => body)
65
- # Parse the RSS feed
66
- feed = RSS::Parser.parse(body)
67
- feed.items.each do |item|
68
- # Put each item into an event
69
- @logger.debug("Item", :item => item.author)
70
- case feed.feed_type
64
+ begin
65
+ feed = RSS::Parser.parse(body)
66
+ feed.items.each do |item|
67
+ # Put each item into an event
68
+ @logger.debug("Item", :item => item.author)
69
+ case feed.feed_type
71
70
  when 'rss'
72
- @codec.decode(item.description) do |event|
73
- event["Feed"] = @url
74
- event["published"] = item.pubDate
75
- event["title"] = item.title
76
- event["link"] = item.link
77
- event["author"] = item.author
78
- decorate(event)
79
- queue << event
80
- end
71
+ handle_rss_response(queue, item)
81
72
  when 'atom'
82
- if ! item.content.nil?
83
- content = item.content.content
84
- else
85
- content = item.summary.content
86
- end
87
- @codec.decode(content) do |event|
88
- event["Feed"] = @url
89
- event["updated"] = item.updated.content
90
- event["title"] = item.title.content
91
- event["link"] = item.link.href
92
- event["author"] = item.author.name.content
93
- unless item.published.nil?
94
- event["published"] = item.published.content
95
- end
96
- decorate(event)
97
- queue << event
98
- end
73
+ handle_atom_response(queue, item)
74
+ end
99
75
  end
76
+ rescue RSS::MissingTagError => e
77
+ @logger.error("Invalid RSS feed", :exception => e)
78
+ rescue => e
79
+ @logger.error("Uknown error while parsing the feed", :url => url, :exception => e)
100
80
  end
101
81
  end
102
82
 
103
83
  def stop
104
84
  Stud.stop!(@run_thread) if @run_thread
105
85
  end
86
+
87
+ private
88
+
89
+ def handle_atom_response(queue, item)
90
+ if ! item.content.nil?
91
+ content = item.content.content
92
+ else
93
+ content = item.summary.content
94
+ end
95
+ @codec.decode(content) do |event|
96
+ event.set("Feed", @url)
97
+ event.set("updated", item.updated.content)
98
+ event.set("title", item.title.content)
99
+ event.set("link", item.link.href)
100
+ ##
101
+ # Author is actually a recommended field, not not a mandatory
102
+ # one, see https://validator.w3.org/feed/docs/atom.html for details.
103
+ ##
104
+ event.set("author", item.author.name.content) if !item.author.nil?
105
+ event.set("published", item.published.content) if !item.published.nil?
106
+
107
+ decorate(event)
108
+ queue << event
109
+ end
110
+ end
111
+ def handle_rss_response(queue, item)
112
+ @codec.decode(item.description) do |event|
113
+ event.set("Feed", @url)
114
+ event.set("published", item.pubDate)
115
+ event.set("title", item.title)
116
+ event.set("link", item.link)
117
+ event.set("author", item.author)
118
+ decorate(event)
119
+ queue << event
120
+ end
121
+ end
106
122
  end # class LogStash::Inputs::Exec
@@ -1,10 +1,10 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-rss'
4
- s.version = '2.0.5'
4
+ s.version = '3.0.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Poll an RSS/Atom feed."
7
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ["Elastic"]
9
9
  s.email = 'jason.kendall@elastic.co'
10
10
  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
20
20
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
21
21
 
22
22
  # Gem dependencies
23
- s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
23
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
24
24
  s.add_runtime_dependency 'logstash-codec-plain'
25
25
  s.add_runtime_dependency 'addressable'
26
26
  s.add_runtime_dependency 'faraday'
@@ -0,0 +1,3 @@
1
+ <?xml version="1.0" encoding="utf-8"?>
2
+ <feed xmlns="http://www.w3.org/2005/Atom">
3
+ </feed>
@@ -0,0 +1,19 @@
1
+ <?xml version="1.0" encoding="utf-8"?>
2
+ <feed xmlns="http://www.w3.org/2005/Atom">
3
+ <title>Example Feed</title>
4
+ <link href="http://example.org/"/>
5
+ <updated>2003-12-13T18:30:02Z</updated>
6
+ <author>
7
+ <name>John Doe</name>
8
+ </author>
9
+ <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
10
+
11
+ <entry>
12
+ <title>Atom-Powered Robots Run Amok</title>
13
+ <link href="http://example.org/2003/12/13/atom03"/>
14
+ <id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
15
+ <updated>2003-12-13T18:30:02Z</updated>
16
+ <summary>Some text.</summary>
17
+ </entry>
18
+
19
+ </feed>
@@ -0,0 +1,10 @@
1
+ <?xml version="1.0" encoding="utf-8"?>
2
+ <feed xmlns="http://www.w3.org/2005/Atom">
3
+ <title>Example Feed</title>
4
+ <link href="http://example.org/"/>
5
+ <updated>2003-12-13T18:30:02Z</updated>
6
+ <author>
7
+ <name>John Doe</name>
8
+ </author>
9
+ <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
10
+ </feed>
@@ -0,0 +1,5 @@
1
+ <?xml version="1.0" encoding="windows-1252"?>
2
+ <rss version="2.0">
3
+ <channel>
4
+ </channel>
5
+ </rss>
@@ -0,0 +1,52 @@
1
+ <?xml version="1.0" encoding="windows-1252"?>
2
+ <rss version="2.0">
3
+ <channel>
4
+ <title>Sample Feed - Favorite RSS Related Software &amp; Resources</title>
5
+ <description>Take a look at some of FeedForAll&apos;s favorite software and resources for learning more about RSS.</description>
6
+ <link>http://www.feedforall.com</link>
7
+ <category domain="www.dmoz.com">Computers/Software/Internet/Site Management/Content Management</category>
8
+ <copyright>Copyright 2004 NotePage, Inc.</copyright>
9
+ <docs>http://blogs.law.harvard.edu/tech/rss</docs>
10
+ <language>en-us</language>
11
+ <lastBuildDate>Mon, 1 Nov 2004 13:17:17 -0500</lastBuildDate>
12
+ <managingEditor>marketing@feedforall.com</managingEditor>
13
+ <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
14
+ <webMaster>webmaster@feedforall.com</webMaster>
15
+ <generator>FeedForAll Beta1 (0.0.1.8)</generator>
16
+ <image>
17
+ <url>http://www.feedforall.com/feedforall-temp.gif</url>
18
+ <title>FeedForAll Sample Feed</title>
19
+ <link>http://www.feedforall.com/industry-solutions.htm</link>
20
+ <description>FeedForAll Sample Feed</description>
21
+ <width>144</width>
22
+ <height>117</height>
23
+ </image>
24
+ <item>
25
+ <title>RSS Resources</title>
26
+ <description>Be sure to take a look at some of our favorite RSS Resources&lt;br&gt;
27
+ &lt;a href=&quot;http://www.rss-specifications.com&quot;&gt;RSS Specifications&lt;/a&gt;&lt;br&gt;
28
+ &lt;a href=&quot;http://www.blog-connection.com&quot;&gt;Blog Connection&lt;/a&gt;&lt;br&gt;
29
+ &lt;br&gt;</description>
30
+ <link>http://www.feedforall.com</link>
31
+ <pubDate>Tue, 26 Oct 2004 14:01:01 -0500</pubDate>
32
+ </item>
33
+ <item>
34
+ <title>Recommended Desktop Feed Reader Software</title>
35
+ <description>&lt;b&gt;FeedDemon&lt;/b&gt; enables you to quickly read and gather information from hundreds of web sites - without having to visit them. Don&apos;t waste any more time checking your favorite web sites for updates. Instead, use FeedDemon and make them come to you. &lt;br&gt;
36
+ More &lt;a href=&quot;http://store.esellerate.net/a.asp?c=1_SKU5139890208_AFL403073819&quot;&gt;FeedDemon Information&lt;/a&gt;</description>
37
+ <link>http://www.feedforall.com/feedforall-partners.htm</link>
38
+ <pubDate>Tue, 26 Oct 2004 14:03:25 -0500</pubDate>
39
+ </item>
40
+ <item>
41
+ <title>Recommended Web Based Feed Reader Software</title>
42
+ <description>&lt;b&gt;FeedScout&lt;/b&gt; enables you to view RSS/ATOM/RDF feeds from different sites directly in Internet Explorer. You can even set your Home Page to show favorite feeds. Feed Scout is a plug-in for Internet Explorer, so you won&apos;t have to learn anything except for how to press 2 new buttons on Internet Explorer toolbar. &lt;br&gt;
43
+ More &lt;a href=&quot;http://www.bytescout.com/feedscout.html&quot;&gt;Information on FeedScout&lt;/a&gt;&lt;br&gt;
44
+ &lt;br&gt;
45
+ &lt;br&gt;
46
+ &lt;b&gt;SurfPack&lt;/b&gt; can feature search tools, horoscopes, current weather conditions, LiveJournal diaries, humor, web modules and other dynamically updated content. &lt;br&gt;
47
+ More &lt;a href=&quot;http://www.surfpack.com/&quot;&gt;Information on SurfPack&lt;/a&gt;&lt;br&gt;</description>
48
+ <link>http://www.feedforall.com/feedforall-partners.htm</link>
49
+ <pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
50
+ </item>
51
+ </channel>
52
+ </rss>
@@ -0,0 +1,8 @@
1
+ <?xml version="1.0" encoding="windows-1252"?>
2
+ <rss version="2.0">
3
+ <channel>
4
+ <title>Sample Feed - Favorite RSS Related Software &amp; Resources</title>
5
+ <description>Take a look at some of FeedForAll&apos;s favorite software and resources for learning more about RSS.</description>
6
+ <link>http://www.feedforall.com</link>
7
+ </channel>
8
+ </rss>
@@ -1,6 +1,7 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/inputs/rss"
4
+ require 'ostruct'
4
5
 
5
6
  describe LogStash::Inputs::Rss do
6
7
  describe "stopping" do
@@ -11,4 +12,96 @@ describe LogStash::Inputs::Rss do
11
12
  end
12
13
  it_behaves_like "an interruptible input plugin"
13
14
  end
14
- end
15
+
16
+ shared_examples "fetching data" do |type|
17
+ let(:config) do
18
+ {
19
+ "url" => "http://www.example.com/foo.rss",
20
+ "interval" => 10
21
+ }
22
+ end
23
+
24
+ let(:sample) do
25
+ body = File.read(File.join(fixtures_source, "sample-feed.xml"))
26
+ OpenStruct.new(:body => body)
27
+ end
28
+
29
+ before(:each) do
30
+ allow(Faraday).to receive(:get).with(config["url"]).and_return(sample)
31
+ end
32
+
33
+ context "when the feed is valid" do
34
+ let(:data) do
35
+ plugin = described_class.new(config)
36
+ plugin_input(plugin) do |queue|
37
+ sleep 0.1 while queue.empty?
38
+ events = []
39
+ queue.size.times { |i| events << queue.pop }
40
+ events
41
+ end
42
+ end
43
+
44
+ it "fetchs all items" do
45
+ expect(data.count).to be > 0
46
+ end
47
+ end
48
+
49
+ context "when the feed is invalid" do
50
+
51
+ let(:sample) do
52
+ body = File.read(File.join(fixtures_source, "invalid-feed.xml"))
53
+ OpenStruct.new(:body => body)
54
+ end
55
+
56
+ let(:plugin) { described_class.new(config) }
57
+
58
+ it "fetchs no items and causes no errors" do
59
+ events = []
60
+ expect {
61
+ plugin_input(plugin) do |queue|
62
+ sleep 1
63
+ events = []
64
+ queue.size.times { |i| events << queue.pop }
65
+ events
66
+ end
67
+ }.not_to raise_error
68
+ expect(events.count).to be == 0
69
+ end
70
+ end
71
+
72
+ context "when the feed is valid, but has zero items" do
73
+
74
+ let(:sample) do
75
+ body = File.read(File.join(fixtures_source, "zero-items-feed.xml"))
76
+ OpenStruct.new(:body => body)
77
+ end
78
+
79
+ let(:plugin) { described_class.new(config) }
80
+
81
+ it "fetchs no items and causes no errors" do
82
+ events = []
83
+ expect {
84
+ plugin_input(plugin) do |queue|
85
+ sleep 1
86
+ events = []
87
+ queue.size.times { |i| events << queue.pop }
88
+ events
89
+ end
90
+ }.not_to raise_error
91
+ expect(events.count).to be == 0
92
+ end
93
+ end
94
+
95
+ end
96
+
97
+ describe "rss feed" do
98
+ let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "rss") }
99
+ it_behaves_like "fetching data"
100
+ end
101
+
102
+ describe "atom feed" do
103
+ let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "atom") }
104
+ it_behaves_like "fetching data"
105
+ end
106
+
107
+ end
metadata CHANGED
@@ -1,21 +1,21 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.5
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-24 00:00:00.000000000 Z
11
+ date: 2016-07-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
16
  - - "~>"
17
17
  - !ruby/object:Gem::Version
18
- version: '1.0'
18
+ version: '2.0'
19
19
  name: logstash-core-plugin-api
20
20
  prerelease: false
21
21
  type: :runtime
@@ -23,7 +23,7 @@ dependencies:
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.0'
26
+ version: '2.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
@@ -94,7 +94,7 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
- description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
97
+ description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
98
98
  email: jason.kendall@elastic.co
99
99
  executables: []
100
100
  extensions: []
@@ -108,6 +108,12 @@ files:
108
108
  - README.md
109
109
  - lib/logstash/inputs/rss.rb
110
110
  - logstash-input-rss.gemspec
111
+ - spec/fixtures/atom/invalid-feed.xml
112
+ - spec/fixtures/atom/sample-feed.xml
113
+ - spec/fixtures/atom/zero-items-feed.xml
114
+ - spec/fixtures/rss/invalid-feed.xml
115
+ - spec/fixtures/rss/sample-feed.xml
116
+ - spec/fixtures/rss/zero-items-feed.xml
111
117
  - spec/inputs/rss_spec.rb
112
118
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
113
119
  licenses:
@@ -136,4 +142,10 @@ signing_key:
136
142
  specification_version: 4
137
143
  summary: Poll an RSS/Atom feed.
138
144
  test_files:
145
+ - spec/fixtures/atom/invalid-feed.xml
146
+ - spec/fixtures/atom/sample-feed.xml
147
+ - spec/fixtures/atom/zero-items-feed.xml
148
+ - spec/fixtures/rss/invalid-feed.xml
149
+ - spec/fixtures/rss/sample-feed.xml
150
+ - spec/fixtures/rss/zero-items-feed.xml
139
151
  - spec/inputs/rss_spec.rb