logstash-input-rss 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: f9b3d490df24bd5775433a318639ae49c153959f
4
+ data.tar.gz: 5d99c9b9d094d5fbb42d78b4aefd45d86cd83e58
5
+ SHA512:
6
+ metadata.gz: 2360ecbb3659b0775b4e22234448cb95738ff485c0c160e1b11d006654122e30c55802c6416b69010321fa271e3c77e6707e3d6318545e926d4c51ccc4894475
7
+ data.tar.gz: a73574640dc941336bcd217e8ccacd73c3a17c69ec83821d673a0627a7e7f386e8794db671049973d2a64d3e7a1b254360684d979a3d2da0448d68ed200b5783
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+ gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,7 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
7
+ require "logstash/devutils/rake"
@@ -0,0 +1,96 @@
1
+ # encoding: utf-8
2
+ require "logstash/inputs/base"
3
+ require "logstash/namespace"
4
+ require "socket" # for Socket.gethostname
5
+
6
+ # Run command line tools and capture the whole output as an event.
7
+ #
8
+ # Notes:
9
+ #
10
+ # * The `@source` of this event will be the command run.
11
+ # * The `@message` of this event will be the entire stdout of the command
12
+ # as one event.
13
+ #
14
+ class LogStash::Inputs::Rss < LogStash::Inputs::Base
15
+
16
+ config_name "rss"
17
+ milestone 2
18
+
19
+ default :codec, "plain"
20
+
21
+ # RSS/Atom feed URL
22
+ config :url, :validate => :string, :required => true
23
+
24
+ # Interval to run the command. Value is in seconds.
25
+ config :interval, :validate => :number, :required => true
26
+
27
+ public
28
+ def register
29
+ require "faraday"
30
+ require "rss"
31
+ @logger.info("Registering RSS Input", :url => @url, :interval => @interval)
32
+ end # def register
33
+
34
+ public
35
+ def run(queue)
36
+ loop do
37
+ start = Time.now
38
+ @logger.info? && @logger.info("Polling RSS", :url => @url)
39
+
40
+ # Pull down the RSS feed using FTW so we can make use of future cache functions
41
+ response = Faraday.get @url
42
+ body = response.body
43
+ # @logger.debug("Body", :body => body)
44
+ # Parse the RSS feed
45
+ feed = RSS::Parser.parse(body)
46
+ feed.items.each do |item|
47
+ # Put each item into an event
48
+ @logger.debug("Item", :item => item.author)
49
+ case feed.feed_type
50
+ when 'rss'
51
+ @codec.decode(item.description) do |event|
52
+ event["Feed"] = @url
53
+ event["published"] = item.pubDate
54
+ event["title"] = item.title
55
+ event["link"] = item.link
56
+ event["author"] = item.author
57
+ decorate(event)
58
+ queue << event
59
+ end
60
+ when 'atom'
61
+ if ! item.content.nil?
62
+ content = item.content.content
63
+ else
64
+ content = item.summary.content
65
+ end
66
+ @codec.decode(content) do |event|
67
+ event["Feed"] = @url
68
+ event["updated"] = item.updated.content
69
+ event["title"] = item.title.content
70
+ event["link"] = item.link.href
71
+ event["author"] = item.author.name.content
72
+ unless item.published.nil?
73
+ event["published"] = item.published.content
74
+ end
75
+ decorate(event)
76
+ queue << event
77
+ end
78
+ end
79
+ end
80
+ duration = Time.now - start
81
+ @logger.info? && @logger.info("Command completed", :command => @command,
82
+ :duration => duration)
83
+
84
+ # Sleep for the remainder of the interval, or 0 if the duration ran
85
+ # longer than the interval.
86
+ sleeptime = [0, @interval - duration].max
87
+ if sleeptime == 0
88
+ @logger.warn("Execution ran longer than the interval. Skipping sleep.",
89
+ :command => @command, :duration => duration,
90
+ :interval => @interval)
91
+ else
92
+ sleep(sleeptime)
93
+ end
94
+ end # loop
95
+ end # def run
96
+ end # class LogStash::Inputs::Exec
@@ -0,0 +1,30 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-input-rss'
4
+ s.version = '0.1.1'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = "Poll an RSS/Atom feed."
7
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'jason.kendall@elasticsearch.com'
10
+ s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+
25
+ s.add_runtime_dependency 'logstash-codec-plain'
26
+ s.add_runtime_dependency 'addressable'
27
+
28
+ s.add_development_dependency 'logstash-devutils'
29
+ end
30
+
File without changes
metadata ADDED
@@ -0,0 +1,114 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-rss
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-19 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ requirement: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - '>='
26
+ - !ruby/object:Gem::Version
27
+ version: 1.4.0
28
+ - - <
29
+ - !ruby/object:Gem::Version
30
+ version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
+ - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-plain
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ requirement: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - '>='
43
+ - !ruby/object:Gem::Version
44
+ version: '0'
45
+ prerelease: false
46
+ type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: addressable
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ requirement: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - '>='
57
+ - !ruby/object:Gem::Version
58
+ version: '0'
59
+ prerelease: false
60
+ type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
63
+ version_requirements: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - '>='
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ requirement: !ruby/object:Gem::Requirement
69
+ requirements:
70
+ - - '>='
71
+ - !ruby/object:Gem::Version
72
+ version: '0'
73
+ prerelease: false
74
+ type: :development
75
+ description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
76
+ email: jason.kendall@elasticsearch.com
77
+ executables: []
78
+ extensions: []
79
+ extra_rdoc_files: []
80
+ files:
81
+ - Gemfile
82
+ - LICENSE
83
+ - Rakefile
84
+ - lib/logstash/inputs/rss.rb
85
+ - logstash-input-rss.gemspec
86
+ - spec/inputs/rss_spec.rb
87
+ homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
88
+ licenses:
89
+ - Apache License (2.0)
90
+ metadata:
91
+ logstash_plugin: 'true'
92
+ logstash_group: input
93
+ post_install_message:
94
+ rdoc_options: []
95
+ require_paths:
96
+ - lib
97
+ required_ruby_version: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - '>='
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ required_rubygems_version: !ruby/object:Gem::Requirement
103
+ requirements:
104
+ - - '>='
105
+ - !ruby/object:Gem::Version
106
+ version: '0'
107
+ requirements: []
108
+ rubyforge_project:
109
+ rubygems_version: 2.2.2
110
+ signing_key:
111
+ specification_version: 4
112
+ summary: Poll an RSS/Atom feed.
113
+ test_files:
114
+ - spec/inputs/rss_spec.rb