logstash-input-rss 2.0.5 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -2
- data/LICENSE +1 -1
- data/README.md +12 -3
- data/lib/logstash/inputs/rss.rb +49 -33
- data/logstash-input-rss.gemspec +3 -3
- data/spec/fixtures/atom/invalid-feed.xml +3 -0
- data/spec/fixtures/atom/sample-feed.xml +19 -0
- data/spec/fixtures/atom/zero-items-feed.xml +10 -0
- data/spec/fixtures/rss/invalid-feed.xml +5 -0
- data/spec/fixtures/rss/sample-feed.xml +52 -0
- data/spec/fixtures/rss/zero-items-feed.xml +8 -0
- data/spec/inputs/rss_spec.rb +94 -1
- metadata +17 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c3c462cbab4cecbdfe73ffcc3113fd9bdcff69a0
|
4
|
+
data.tar.gz: b4d8690696754c3fc745589a39678a5ebd5e7d95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eef2d87ebc599e7789c168182970083f40b52ece6c2918b62f3efe14c31bcb8a25e8ee9633187f29192ea525f51fa8328f6b4adadfdbd06106790ff3127ba3cf
|
7
|
+
data.tar.gz: 1e3810dccfc333b47f3c99f9318bc8a8b1cdc84dd43943ffe8daa7ca55beb522cf2de501fc7dfa888cc15aead182d67beab80f817cb8da2cb7fe7f0e2094d0f7
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,17 @@
|
|
1
|
-
|
1
|
+
## 3.0.0
|
2
|
+
- Breaking: Updated plugin to use new Java Event APIs.
|
3
|
+
## 2.0.6
|
4
|
+
- Fixes the case when there is an invalid feed, exceptions are now
|
5
|
+
catched and logged. Fixes https://github.com/logstash-plugins/logstash-input-rss/issues/1
|
6
|
+
- Also add test for normal, valid but empty and invalid feeds use
|
7
|
+
cases.
|
8
|
+
|
9
|
+
## 2.0.5
|
2
10
|
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
3
|
-
|
11
|
+
|
12
|
+
## 2.0.4
|
4
13
|
- New dependency requirements for logstash-core for the 5.0 release
|
14
|
+
|
5
15
|
## 2.0.3
|
6
16
|
- Fixed bug where `queue` variable was being used but not initialized in `handle_response`.
|
7
17
|
Ref: https://github.com/logstash-plugins/logstash-input-rss/pull/14
|
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-rss-unit/)
|
3
|
+
[](https://travis-ci.org/logstash-plugins/logstash-input-rss)
|
5
4
|
|
6
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
7
6
|
|
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
|
|
56
55
|
```
|
57
56
|
- Install plugin
|
58
57
|
```sh
|
58
|
+
# Logstash 2.3 and higher
|
59
|
+
bin/logstash-plugin install --no-verify
|
60
|
+
|
61
|
+
# Prior to Logstash 2.3
|
59
62
|
bin/plugin install --no-verify
|
63
|
+
|
60
64
|
```
|
61
65
|
- Run Logstash with your plugin
|
62
66
|
```sh
|
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
|
|
74
78
|
```
|
75
79
|
- Install the plugin from the Logstash home
|
76
80
|
```sh
|
77
|
-
|
81
|
+
# Logstash 2.3 and higher
|
82
|
+
bin/logstash-plugin install --no-verify
|
83
|
+
|
84
|
+
# Prior to Logstash 2.3
|
85
|
+
bin/plugin install --no-verify
|
86
|
+
|
78
87
|
```
|
79
88
|
- Start Logstash and proceed to test the plugin
|
80
89
|
|
data/lib/logstash/inputs/rss.rb
CHANGED
@@ -61,46 +61,62 @@ class LogStash::Inputs::Rss < LogStash::Inputs::Base
|
|
61
61
|
|
62
62
|
def handle_response(response, queue)
|
63
63
|
body = response.body
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
case feed.feed_type
|
64
|
+
begin
|
65
|
+
feed = RSS::Parser.parse(body)
|
66
|
+
feed.items.each do |item|
|
67
|
+
# Put each item into an event
|
68
|
+
@logger.debug("Item", :item => item.author)
|
69
|
+
case feed.feed_type
|
71
70
|
when 'rss'
|
72
|
-
|
73
|
-
event["Feed"] = @url
|
74
|
-
event["published"] = item.pubDate
|
75
|
-
event["title"] = item.title
|
76
|
-
event["link"] = item.link
|
77
|
-
event["author"] = item.author
|
78
|
-
decorate(event)
|
79
|
-
queue << event
|
80
|
-
end
|
71
|
+
handle_rss_response(queue, item)
|
81
72
|
when 'atom'
|
82
|
-
|
83
|
-
|
84
|
-
else
|
85
|
-
content = item.summary.content
|
86
|
-
end
|
87
|
-
@codec.decode(content) do |event|
|
88
|
-
event["Feed"] = @url
|
89
|
-
event["updated"] = item.updated.content
|
90
|
-
event["title"] = item.title.content
|
91
|
-
event["link"] = item.link.href
|
92
|
-
event["author"] = item.author.name.content
|
93
|
-
unless item.published.nil?
|
94
|
-
event["published"] = item.published.content
|
95
|
-
end
|
96
|
-
decorate(event)
|
97
|
-
queue << event
|
98
|
-
end
|
73
|
+
handle_atom_response(queue, item)
|
74
|
+
end
|
99
75
|
end
|
76
|
+
rescue RSS::MissingTagError => e
|
77
|
+
@logger.error("Invalid RSS feed", :exception => e)
|
78
|
+
rescue => e
|
79
|
+
@logger.error("Uknown error while parsing the feed", :url => url, :exception => e)
|
100
80
|
end
|
101
81
|
end
|
102
82
|
|
103
83
|
def stop
|
104
84
|
Stud.stop!(@run_thread) if @run_thread
|
105
85
|
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
def handle_atom_response(queue, item)
|
90
|
+
if ! item.content.nil?
|
91
|
+
content = item.content.content
|
92
|
+
else
|
93
|
+
content = item.summary.content
|
94
|
+
end
|
95
|
+
@codec.decode(content) do |event|
|
96
|
+
event.set("Feed", @url)
|
97
|
+
event.set("updated", item.updated.content)
|
98
|
+
event.set("title", item.title.content)
|
99
|
+
event.set("link", item.link.href)
|
100
|
+
##
|
101
|
+
# Author is actually a recommended field, not not a mandatory
|
102
|
+
# one, see https://validator.w3.org/feed/docs/atom.html for details.
|
103
|
+
##
|
104
|
+
event.set("author", item.author.name.content) if !item.author.nil?
|
105
|
+
event.set("published", item.published.content) if !item.published.nil?
|
106
|
+
|
107
|
+
decorate(event)
|
108
|
+
queue << event
|
109
|
+
end
|
110
|
+
end
|
111
|
+
def handle_rss_response(queue, item)
|
112
|
+
@codec.decode(item.description) do |event|
|
113
|
+
event.set("Feed", @url)
|
114
|
+
event.set("published", item.pubDate)
|
115
|
+
event.set("title", item.title)
|
116
|
+
event.set("link", item.link)
|
117
|
+
event.set("author", item.author)
|
118
|
+
decorate(event)
|
119
|
+
queue << event
|
120
|
+
end
|
121
|
+
end
|
106
122
|
end # class LogStash::Inputs::Exec
|
data/logstash-input-rss.gemspec
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-rss'
|
4
|
-
s.version = '
|
4
|
+
s.version = '3.0.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Poll an RSS/Atom feed."
|
7
|
-
s.description = "This gem is a
|
7
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
8
8
|
s.authors = ["Elastic"]
|
9
9
|
s.email = 'jason.kendall@elastic.co'
|
10
10
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
|
-
s.add_runtime_dependency "logstash-core-plugin-api", "~>
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
24
24
|
s.add_runtime_dependency 'logstash-codec-plain'
|
25
25
|
s.add_runtime_dependency 'addressable'
|
26
26
|
s.add_runtime_dependency 'faraday'
|
@@ -0,0 +1,19 @@
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?>
|
2
|
+
<feed xmlns="http://www.w3.org/2005/Atom">
|
3
|
+
<title>Example Feed</title>
|
4
|
+
<link href="http://example.org/"/>
|
5
|
+
<updated>2003-12-13T18:30:02Z</updated>
|
6
|
+
<author>
|
7
|
+
<name>John Doe</name>
|
8
|
+
</author>
|
9
|
+
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
|
10
|
+
|
11
|
+
<entry>
|
12
|
+
<title>Atom-Powered Robots Run Amok</title>
|
13
|
+
<link href="http://example.org/2003/12/13/atom03"/>
|
14
|
+
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
15
|
+
<updated>2003-12-13T18:30:02Z</updated>
|
16
|
+
<summary>Some text.</summary>
|
17
|
+
</entry>
|
18
|
+
|
19
|
+
</feed>
|
@@ -0,0 +1,10 @@
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?>
|
2
|
+
<feed xmlns="http://www.w3.org/2005/Atom">
|
3
|
+
<title>Example Feed</title>
|
4
|
+
<link href="http://example.org/"/>
|
5
|
+
<updated>2003-12-13T18:30:02Z</updated>
|
6
|
+
<author>
|
7
|
+
<name>John Doe</name>
|
8
|
+
</author>
|
9
|
+
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
|
10
|
+
</feed>
|
@@ -0,0 +1,52 @@
|
|
1
|
+
<?xml version="1.0" encoding="windows-1252"?>
|
2
|
+
<rss version="2.0">
|
3
|
+
<channel>
|
4
|
+
<title>Sample Feed - Favorite RSS Related Software & Resources</title>
|
5
|
+
<description>Take a look at some of FeedForAll's favorite software and resources for learning more about RSS.</description>
|
6
|
+
<link>http://www.feedforall.com</link>
|
7
|
+
<category domain="www.dmoz.com">Computers/Software/Internet/Site Management/Content Management</category>
|
8
|
+
<copyright>Copyright 2004 NotePage, Inc.</copyright>
|
9
|
+
<docs>http://blogs.law.harvard.edu/tech/rss</docs>
|
10
|
+
<language>en-us</language>
|
11
|
+
<lastBuildDate>Mon, 1 Nov 2004 13:17:17 -0500</lastBuildDate>
|
12
|
+
<managingEditor>marketing@feedforall.com</managingEditor>
|
13
|
+
<pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
|
14
|
+
<webMaster>webmaster@feedforall.com</webMaster>
|
15
|
+
<generator>FeedForAll Beta1 (0.0.1.8)</generator>
|
16
|
+
<image>
|
17
|
+
<url>http://www.feedforall.com/feedforall-temp.gif</url>
|
18
|
+
<title>FeedForAll Sample Feed</title>
|
19
|
+
<link>http://www.feedforall.com/industry-solutions.htm</link>
|
20
|
+
<description>FeedForAll Sample Feed</description>
|
21
|
+
<width>144</width>
|
22
|
+
<height>117</height>
|
23
|
+
</image>
|
24
|
+
<item>
|
25
|
+
<title>RSS Resources</title>
|
26
|
+
<description>Be sure to take a look at some of our favorite RSS Resources<br>
|
27
|
+
<a href="http://www.rss-specifications.com">RSS Specifications</a><br>
|
28
|
+
<a href="http://www.blog-connection.com">Blog Connection</a><br>
|
29
|
+
<br></description>
|
30
|
+
<link>http://www.feedforall.com</link>
|
31
|
+
<pubDate>Tue, 26 Oct 2004 14:01:01 -0500</pubDate>
|
32
|
+
</item>
|
33
|
+
<item>
|
34
|
+
<title>Recommended Desktop Feed Reader Software</title>
|
35
|
+
<description><b>FeedDemon</b> enables you to quickly read and gather information from hundreds of web sites - without having to visit them. Don't waste any more time checking your favorite web sites for updates. Instead, use FeedDemon and make them come to you. <br>
|
36
|
+
More <a href="http://store.esellerate.net/a.asp?c=1_SKU5139890208_AFL403073819">FeedDemon Information</a></description>
|
37
|
+
<link>http://www.feedforall.com/feedforall-partners.htm</link>
|
38
|
+
<pubDate>Tue, 26 Oct 2004 14:03:25 -0500</pubDate>
|
39
|
+
</item>
|
40
|
+
<item>
|
41
|
+
<title>Recommended Web Based Feed Reader Software</title>
|
42
|
+
<description><b>FeedScout</b> enables you to view RSS/ATOM/RDF feeds from different sites directly in Internet Explorer. You can even set your Home Page to show favorite feeds. Feed Scout is a plug-in for Internet Explorer, so you won't have to learn anything except for how to press 2 new buttons on Internet Explorer toolbar. <br>
|
43
|
+
More <a href="http://www.bytescout.com/feedscout.html">Information on FeedScout</a><br>
|
44
|
+
<br>
|
45
|
+
<br>
|
46
|
+
<b>SurfPack</b> can feature search tools, horoscopes, current weather conditions, LiveJournal diaries, humor, web modules and other dynamically updated content. <br>
|
47
|
+
More <a href="http://www.surfpack.com/">Information on SurfPack</a><br></description>
|
48
|
+
<link>http://www.feedforall.com/feedforall-partners.htm</link>
|
49
|
+
<pubDate>Tue, 26 Oct 2004 14:06:44 -0500</pubDate>
|
50
|
+
</item>
|
51
|
+
</channel>
|
52
|
+
</rss>
|
@@ -0,0 +1,8 @@
|
|
1
|
+
<?xml version="1.0" encoding="windows-1252"?>
|
2
|
+
<rss version="2.0">
|
3
|
+
<channel>
|
4
|
+
<title>Sample Feed - Favorite RSS Related Software & Resources</title>
|
5
|
+
<description>Take a look at some of FeedForAll's favorite software and resources for learning more about RSS.</description>
|
6
|
+
<link>http://www.feedforall.com</link>
|
7
|
+
</channel>
|
8
|
+
</rss>
|
data/spec/inputs/rss_spec.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/inputs/rss"
|
4
|
+
require 'ostruct'
|
4
5
|
|
5
6
|
describe LogStash::Inputs::Rss do
|
6
7
|
describe "stopping" do
|
@@ -11,4 +12,96 @@ describe LogStash::Inputs::Rss do
|
|
11
12
|
end
|
12
13
|
it_behaves_like "an interruptible input plugin"
|
13
14
|
end
|
14
|
-
|
15
|
+
|
16
|
+
shared_examples "fetching data" do |type|
|
17
|
+
let(:config) do
|
18
|
+
{
|
19
|
+
"url" => "http://www.example.com/foo.rss",
|
20
|
+
"interval" => 10
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
let(:sample) do
|
25
|
+
body = File.read(File.join(fixtures_source, "sample-feed.xml"))
|
26
|
+
OpenStruct.new(:body => body)
|
27
|
+
end
|
28
|
+
|
29
|
+
before(:each) do
|
30
|
+
allow(Faraday).to receive(:get).with(config["url"]).and_return(sample)
|
31
|
+
end
|
32
|
+
|
33
|
+
context "when the feed is valid" do
|
34
|
+
let(:data) do
|
35
|
+
plugin = described_class.new(config)
|
36
|
+
plugin_input(plugin) do |queue|
|
37
|
+
sleep 0.1 while queue.empty?
|
38
|
+
events = []
|
39
|
+
queue.size.times { |i| events << queue.pop }
|
40
|
+
events
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
it "fetchs all items" do
|
45
|
+
expect(data.count).to be > 0
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
context "when the feed is invalid" do
|
50
|
+
|
51
|
+
let(:sample) do
|
52
|
+
body = File.read(File.join(fixtures_source, "invalid-feed.xml"))
|
53
|
+
OpenStruct.new(:body => body)
|
54
|
+
end
|
55
|
+
|
56
|
+
let(:plugin) { described_class.new(config) }
|
57
|
+
|
58
|
+
it "fetchs no items and causes no errors" do
|
59
|
+
events = []
|
60
|
+
expect {
|
61
|
+
plugin_input(plugin) do |queue|
|
62
|
+
sleep 1
|
63
|
+
events = []
|
64
|
+
queue.size.times { |i| events << queue.pop }
|
65
|
+
events
|
66
|
+
end
|
67
|
+
}.not_to raise_error
|
68
|
+
expect(events.count).to be == 0
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context "when the feed is valid, but has zero items" do
|
73
|
+
|
74
|
+
let(:sample) do
|
75
|
+
body = File.read(File.join(fixtures_source, "zero-items-feed.xml"))
|
76
|
+
OpenStruct.new(:body => body)
|
77
|
+
end
|
78
|
+
|
79
|
+
let(:plugin) { described_class.new(config) }
|
80
|
+
|
81
|
+
it "fetchs no items and causes no errors" do
|
82
|
+
events = []
|
83
|
+
expect {
|
84
|
+
plugin_input(plugin) do |queue|
|
85
|
+
sleep 1
|
86
|
+
events = []
|
87
|
+
queue.size.times { |i| events << queue.pop }
|
88
|
+
events
|
89
|
+
end
|
90
|
+
}.not_to raise_error
|
91
|
+
expect(events.count).to be == 0
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
|
97
|
+
describe "rss feed" do
|
98
|
+
let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "rss") }
|
99
|
+
it_behaves_like "fetching data"
|
100
|
+
end
|
101
|
+
|
102
|
+
describe "atom feed" do
|
103
|
+
let(:fixtures_source) { File.join(File.dirname(__FILE__), "..", "fixtures", "atom") }
|
104
|
+
it_behaves_like "fetching data"
|
105
|
+
end
|
106
|
+
|
107
|
+
end
|
metadata
CHANGED
@@ -1,21 +1,21 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-rss
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
16
|
- - "~>"
|
17
17
|
- !ruby/object:Gem::Version
|
18
|
-
version: '
|
18
|
+
version: '2.0'
|
19
19
|
name: logstash-core-plugin-api
|
20
20
|
prerelease: false
|
21
21
|
type: :runtime
|
@@ -23,7 +23,7 @@ dependencies:
|
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '2.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
29
29
|
requirements:
|
@@ -94,7 +94,7 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
-
description: This gem is a
|
97
|
+
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
98
98
|
email: jason.kendall@elastic.co
|
99
99
|
executables: []
|
100
100
|
extensions: []
|
@@ -108,6 +108,12 @@ files:
|
|
108
108
|
- README.md
|
109
109
|
- lib/logstash/inputs/rss.rb
|
110
110
|
- logstash-input-rss.gemspec
|
111
|
+
- spec/fixtures/atom/invalid-feed.xml
|
112
|
+
- spec/fixtures/atom/sample-feed.xml
|
113
|
+
- spec/fixtures/atom/zero-items-feed.xml
|
114
|
+
- spec/fixtures/rss/invalid-feed.xml
|
115
|
+
- spec/fixtures/rss/sample-feed.xml
|
116
|
+
- spec/fixtures/rss/zero-items-feed.xml
|
111
117
|
- spec/inputs/rss_spec.rb
|
112
118
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
113
119
|
licenses:
|
@@ -136,4 +142,10 @@ signing_key:
|
|
136
142
|
specification_version: 4
|
137
143
|
summary: Poll an RSS/Atom feed.
|
138
144
|
test_files:
|
145
|
+
- spec/fixtures/atom/invalid-feed.xml
|
146
|
+
- spec/fixtures/atom/sample-feed.xml
|
147
|
+
- spec/fixtures/atom/zero-items-feed.xml
|
148
|
+
- spec/fixtures/rss/invalid-feed.xml
|
149
|
+
- spec/fixtures/rss/sample-feed.xml
|
150
|
+
- spec/fixtures/rss/zero-items-feed.xml
|
139
151
|
- spec/inputs/rss_spec.rb
|