logstash-input-http_poller 1.1.2 → 2.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +4 -4
- data/lib/logstash/inputs/http_poller.rb +16 -6
- data/logstash-input-http_poller.gemspec +5 -6
- data/spec/inputs/http_poller_spec.rb +5 -0
- metadata +29 -47
- data/.gitignore +0 -7
- data/Rakefile +0 -1
- data/examples/ha_proxy.conf +0 -43
- data/examples/local_elasticsearch.conf +0 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 00bca71c9b7ea90c67ce4a90dc7f70f5c4b1e849
|
4
|
+
data.tar.gz: ce702eac396ec7c0712b8b8897cdc6ed3dc1c5c6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5669ad934a4e3178812ff7ea59944634d0b0278d8dc5837fa50e25fa3ee07edef7087d318c26ddc4ffb8a7e25e8bbce0e5120777e3685a7c5a7ad231eace7b19
|
7
|
+
data.tar.gz: a647b151a1b1ed31f637c5362938e1a35f43f2ee155f94995e993a09cc491be1fa3f2ebf5834aede19b12253016003f439fd5f63dbe34afc0a5bc841685a66ab
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 2.0.0
|
2
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
|
+
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
|
+
- Dependency on logstash-core update to 2.0
|
5
|
+
|
1
6
|
* 1.1.2
|
2
7
|
- Correctly default to zero connection retries
|
3
8
|
- Revert old ineffective code for connection retries
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
This plugin is based off [logstash-input-rest](https://github.com/maximede/logstash-input-rest) by @maximede.
|
4
4
|
|
5
|
-
This [Logstash](https://github.com/
|
5
|
+
This [Logstash](https://github.com/elastic/logstash) input plugin allows you to call an HTTP API, decode the output of it into event(s), and send them on their merry way.
|
6
6
|
|
7
7
|
The idea behind this plugins came from a need to read springboot metrics endpoint, instead of configuring jmx to monitor my java application memory / gc/ etc.
|
8
8
|
|
@@ -14,10 +14,10 @@ For config examples see `http_poller.rb` in `lib/logstash/inputs/` in this repo.
|
|
14
14
|
|
15
15
|
## Documentation
|
16
16
|
|
17
|
-
Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.
|
17
|
+
Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
|
18
18
|
|
19
19
|
- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
|
20
|
-
- For more asciidoc formatting tips, see the excellent reference here https://github.com/
|
20
|
+
- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
|
21
21
|
|
22
22
|
## Need Help?
|
23
23
|
|
@@ -91,4 +91,4 @@ Programming is not a required skill. Whatever you've seen about open source and
|
|
91
91
|
|
92
92
|
It is more important to the community that you are able to contribute.
|
93
93
|
|
94
|
-
For more information about contributing, see the [CONTRIBUTING](https://github.com/
|
94
|
+
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
@@ -5,11 +5,16 @@ require "logstash/plugin_mixins/http_client"
|
|
5
5
|
require "socket" # for Socket.gethostname
|
6
6
|
require "manticore"
|
7
7
|
|
8
|
-
#
|
8
|
+
# This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and
|
9
|
+
# send them on their merry way. The idea behind this plugins came from a need to read springboot
|
10
|
+
# metrics endpoint, instead of configuring jmx to monitor my java application memory/gc/ etc.
|
9
11
|
#
|
10
|
-
#
|
12
|
+
# ==== Example
|
13
|
+
# Reads from a list of urls and decodes the body of the response with a codec.
|
11
14
|
# The config should look like this:
|
12
15
|
#
|
16
|
+
# [source,ruby]
|
17
|
+
# ----------------------------------
|
13
18
|
# input {
|
14
19
|
# http_poller {
|
15
20
|
# urls => {
|
@@ -40,6 +45,7 @@ require "manticore"
|
|
40
45
|
# codec => rubydebug
|
41
46
|
# }
|
42
47
|
# }
|
48
|
+
# ----------------------------------
|
43
49
|
|
44
50
|
class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
45
51
|
include LogStash::PluginMixins::HttpClient
|
@@ -48,18 +54,17 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
48
54
|
|
49
55
|
default :codec, "json"
|
50
56
|
|
51
|
-
# A Hash of urls in this format : "name" => "url"
|
57
|
+
# A Hash of urls in this format : `"name" => "url"`.
|
52
58
|
# The name and the url will be passed in the outputed event
|
53
|
-
#
|
54
59
|
config :urls, :validate => :hash, :required => true
|
55
60
|
|
56
|
-
# How often
|
61
|
+
# How often (in seconds) the urls will be called
|
57
62
|
config :interval, :validate => :number, :required => true
|
58
63
|
|
59
64
|
# Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
|
60
65
|
config :target, :validate => :string
|
61
66
|
|
62
|
-
# If you'd like to work with the request/response metadata
|
67
|
+
# If you'd like to work with the request/response metadata.
|
63
68
|
# Set this value to the name of the field you'd like to store a nested
|
64
69
|
# hash of metadata.
|
65
70
|
config :metadata_target, :validate => :string, :default => '@metadata'
|
@@ -74,6 +79,10 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
74
79
|
setup_requests!
|
75
80
|
end
|
76
81
|
|
82
|
+
def stop
|
83
|
+
Stud.stop!(@interval_thread) if @interval_thread
|
84
|
+
end
|
85
|
+
|
77
86
|
private
|
78
87
|
def setup_requests!
|
79
88
|
@requests = Hash[@urls.map {|name, url| [name, normalize_request(url)] }]
|
@@ -124,6 +133,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
|
|
124
133
|
|
125
134
|
public
|
126
135
|
def run(queue)
|
136
|
+
@interval_thread = Thread.current
|
127
137
|
Stud.interval(@interval) do
|
128
138
|
run_once(queue)
|
129
139
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-http_poller'
|
3
|
-
s.version = '
|
3
|
+
s.version = '2.0.1'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Poll HTTP endpoints with Logstash."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program."
|
@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
|
|
9
9
|
s.require_paths = ["lib"]
|
10
10
|
|
11
11
|
# Files
|
12
|
-
s.files =
|
12
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
13
13
|
# Tests
|
14
14
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
15
15
|
|
@@ -17,13 +17,12 @@ Gem::Specification.new do |s|
|
|
17
17
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
18
18
|
|
19
19
|
# Gem dependencies
|
20
|
-
s.add_runtime_dependency "logstash-core",
|
20
|
+
s.add_runtime_dependency "logstash-core", ">= 2.0.0.snapshot", "< 3.0.0"
|
21
21
|
s.add_runtime_dependency 'logstash-codec-plain'
|
22
|
-
s.add_runtime_dependency 'logstash-codec-json'
|
23
22
|
s.add_runtime_dependency 'logstash-mixin-http_client', ">= 1.0.1"
|
24
|
-
s.add_runtime_dependency 'stud'
|
25
|
-
s.add_runtime_dependency 'manticore'
|
23
|
+
s.add_runtime_dependency 'stud', "~> 0.0.22"
|
26
24
|
|
25
|
+
s.add_development_dependency 'logstash-codec-json'
|
27
26
|
s.add_development_dependency 'logstash-devutils'
|
28
27
|
s.add_development_dependency 'flores'
|
29
28
|
end
|
metadata
CHANGED
@@ -1,148 +1,130 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-http_poller
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- andrewvc
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-10-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - '>='
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 1.5.0
|
20
|
-
- - <
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 2.0.0
|
23
14
|
requirement: !ruby/object:Gem::Requirement
|
24
15
|
requirements:
|
25
16
|
- - '>='
|
26
17
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
18
|
+
version: 2.0.0.snapshot
|
28
19
|
- - <
|
29
20
|
- !ruby/object:Gem::Version
|
30
|
-
version:
|
21
|
+
version: 3.0.0
|
22
|
+
name: logstash-core
|
31
23
|
prerelease: false
|
32
24
|
type: :runtime
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-codec-plain
|
35
25
|
version_requirements: !ruby/object:Gem::Requirement
|
36
26
|
requirements:
|
37
27
|
- - '>='
|
38
28
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
29
|
+
version: 2.0.0.snapshot
|
30
|
+
- - <
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 3.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
40
34
|
requirement: !ruby/object:Gem::Requirement
|
41
35
|
requirements:
|
42
36
|
- - '>='
|
43
37
|
- !ruby/object:Gem::Version
|
44
38
|
version: '0'
|
39
|
+
name: logstash-codec-plain
|
45
40
|
prerelease: false
|
46
41
|
type: :runtime
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
name: logstash-codec-json
|
49
42
|
version_requirements: !ruby/object:Gem::Requirement
|
50
43
|
requirements:
|
51
44
|
- - '>='
|
52
45
|
- !ruby/object:Gem::Version
|
53
46
|
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
54
48
|
requirement: !ruby/object:Gem::Requirement
|
55
49
|
requirements:
|
56
50
|
- - '>='
|
57
51
|
- !ruby/object:Gem::Version
|
58
|
-
version:
|
52
|
+
version: 1.0.1
|
53
|
+
name: logstash-mixin-http_client
|
59
54
|
prerelease: false
|
60
55
|
type: :runtime
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
|
-
name: logstash-mixin-http_client
|
63
56
|
version_requirements: !ruby/object:Gem::Requirement
|
64
57
|
requirements:
|
65
58
|
- - '>='
|
66
59
|
- !ruby/object:Gem::Version
|
67
60
|
version: 1.0.1
|
61
|
+
- !ruby/object:Gem::Dependency
|
68
62
|
requirement: !ruby/object:Gem::Requirement
|
69
63
|
requirements:
|
70
|
-
- -
|
64
|
+
- - ~>
|
71
65
|
- !ruby/object:Gem::Version
|
72
|
-
version:
|
66
|
+
version: 0.0.22
|
67
|
+
name: stud
|
73
68
|
prerelease: false
|
74
69
|
type: :runtime
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
|
-
name: stud
|
77
70
|
version_requirements: !ruby/object:Gem::Requirement
|
78
71
|
requirements:
|
79
|
-
- -
|
72
|
+
- - ~>
|
80
73
|
- !ruby/object:Gem::Version
|
81
|
-
version:
|
74
|
+
version: 0.0.22
|
75
|
+
- !ruby/object:Gem::Dependency
|
82
76
|
requirement: !ruby/object:Gem::Requirement
|
83
77
|
requirements:
|
84
78
|
- - '>='
|
85
79
|
- !ruby/object:Gem::Version
|
86
80
|
version: '0'
|
81
|
+
name: logstash-codec-json
|
87
82
|
prerelease: false
|
88
|
-
type: :
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
|
-
name: manticore
|
83
|
+
type: :development
|
91
84
|
version_requirements: !ruby/object:Gem::Requirement
|
92
85
|
requirements:
|
93
86
|
- - '>='
|
94
87
|
- !ruby/object:Gem::Version
|
95
88
|
version: '0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
96
90
|
requirement: !ruby/object:Gem::Requirement
|
97
91
|
requirements:
|
98
92
|
- - '>='
|
99
93
|
- !ruby/object:Gem::Version
|
100
94
|
version: '0'
|
101
|
-
prerelease: false
|
102
|
-
type: :runtime
|
103
|
-
- !ruby/object:Gem::Dependency
|
104
95
|
name: logstash-devutils
|
96
|
+
prerelease: false
|
97
|
+
type: :development
|
105
98
|
version_requirements: !ruby/object:Gem::Requirement
|
106
99
|
requirements:
|
107
100
|
- - '>='
|
108
101
|
- !ruby/object:Gem::Version
|
109
102
|
version: '0'
|
103
|
+
- !ruby/object:Gem::Dependency
|
110
104
|
requirement: !ruby/object:Gem::Requirement
|
111
105
|
requirements:
|
112
106
|
- - '>='
|
113
107
|
- !ruby/object:Gem::Version
|
114
108
|
version: '0'
|
109
|
+
name: flores
|
115
110
|
prerelease: false
|
116
111
|
type: :development
|
117
|
-
- !ruby/object:Gem::Dependency
|
118
|
-
name: flores
|
119
112
|
version_requirements: !ruby/object:Gem::Requirement
|
120
113
|
requirements:
|
121
114
|
- - '>='
|
122
115
|
- !ruby/object:Gem::Version
|
123
116
|
version: '0'
|
124
|
-
requirement: !ruby/object:Gem::Requirement
|
125
|
-
requirements:
|
126
|
-
- - '>='
|
127
|
-
- !ruby/object:Gem::Version
|
128
|
-
version: '0'
|
129
|
-
prerelease: false
|
130
|
-
type: :development
|
131
117
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program.
|
132
118
|
email:
|
133
119
|
executables: []
|
134
120
|
extensions: []
|
135
121
|
extra_rdoc_files: []
|
136
122
|
files:
|
137
|
-
- .gitignore
|
138
123
|
- CHANGELOG.md
|
139
124
|
- DEVELOPER.md
|
140
125
|
- Gemfile
|
141
126
|
- LICENSE
|
142
127
|
- README.md
|
143
|
-
- Rakefile
|
144
|
-
- examples/ha_proxy.conf
|
145
|
-
- examples/local_elasticsearch.conf
|
146
128
|
- lib/logstash/inputs/http_poller.rb
|
147
129
|
- logstash-input-http_poller.gemspec
|
148
130
|
- spec/inputs/http_poller_spec.rb
|
@@ -168,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
168
150
|
version: '0'
|
169
151
|
requirements: []
|
170
152
|
rubyforge_project:
|
171
|
-
rubygems_version: 2.
|
153
|
+
rubygems_version: 2.4.8
|
172
154
|
signing_key:
|
173
155
|
specification_version: 4
|
174
156
|
summary: Poll HTTP endpoints with Logstash.
|
data/.gitignore
DELETED
data/Rakefile
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "logstash/devutils/rake"
|
data/examples/ha_proxy.conf
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
input {
|
2
|
-
http_poller {
|
3
|
-
urls => {
|
4
|
-
haproxy => {
|
5
|
-
method => get
|
6
|
-
url => "http://localhost:9201/haproxy?stats;csv"
|
7
|
-
auth => {
|
8
|
-
user => "myuser"
|
9
|
-
password => "mypass"
|
10
|
-
}
|
11
|
-
}
|
12
|
-
}
|
13
|
-
request_timeout => 60
|
14
|
-
interval => 60
|
15
|
-
codec => "plain" # This is CSV data, so just let it be plain for now
|
16
|
-
metadata_target => "_http_poller_metadata"
|
17
|
-
}
|
18
|
-
}
|
19
|
-
|
20
|
-
filter {
|
21
|
-
split {
|
22
|
-
}
|
23
|
-
|
24
|
-
csv {
|
25
|
-
columns => [ pxname,svname,qcur,qmax,scur,smax,slim,stot,binhttp://localhost:9201/haproxy?stats;csvm,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime ]
|
26
|
-
}
|
27
|
-
|
28
|
-
# This is a pointless CSV header
|
29
|
-
if [pxname] == "# pxname" {
|
30
|
-
drop{}
|
31
|
-
}
|
32
|
-
}
|
33
|
-
|
34
|
-
output {
|
35
|
-
stdout {
|
36
|
-
codec => rubydebug
|
37
|
-
}
|
38
|
-
|
39
|
-
elasticsearch {
|
40
|
-
protocol => "http"
|
41
|
-
cluster => "avc-cluster"
|
42
|
-
}
|
43
|
-
}
|