logstash-mixin-http_client 2.0.3 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -4
- data/README.md +6 -1
- data/lib/logstash/plugin_mixins/http_client.rb +10 -3
- data/lib/logstash/plugin_mixins/http_client.rb~ +163 -0
- data/logstash-mixin-http_client.gemspec +1 -1
- metadata +17 -16
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3410c3bea0f912007ff5ca1c641ae110720ca381
|
|
4
|
+
data.tar.gz: 61ac66897e399787cf7cf15d74119e39a6ba9be7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 085c721ae870334c7633f1cc19f605b6cc656b46ecf1dcf605252a996cce699e5bd44583fa55fa35e653bfd6442229a998d11384ef49c754273c4aaffdd9bf09
|
|
7
|
+
data.tar.gz: 8257554a0bac0ca4dc6b6a7a4681a7ce4a2b54c18b209dd5625333661df7cab220d5f036b452a73d54b9884aaf794a9a0a155844709547996fce918ab95016e9
|
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
# 2.1.0
|
|
2
|
+
* Default `automatic_retries` to 1 to fix connections to hosts with broken keepalive
|
|
3
|
+
* Add `non_idempotent_retries` option
|
|
4
|
+
# 2.0.0
|
|
5
|
+
* Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
|
3
6
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
|
4
|
-
|
|
5
|
-
|
|
7
|
+
* Dependency on logstash-core update to 2.0
|
|
6
8
|
# 1.0.2
|
|
7
9
|
* Add 'verify_cert' config option
|
|
8
10
|
# 1.0.1
|
data/README.md
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
# Easily Add an HTTP Client to your Logstash Plugin!
|
|
2
2
|
|
|
3
|
+
[](https://travis-ci.org/logstash-plugins/logstash-mixin-http_client)
|
|
4
|
+
|
|
3
5
|
HTTP clients have a lot of configurable options (proxies, certificates, headers, etc.), and specifying all of these with proper validation for a logstash plugin can be irritating. We built this plugin while building our [HTTP Poller Input](https://github.com/logstash-plugins/logstash-input-http_poller). If you need to build a plugin that works primarily based around HTTP this mixin makes it easy and consistent! It is based on [Manticore](https://github.com/cheald/manticore) a lightning quick, fully featured JRuby HTTP client based on Apache Commons HTTP Client.`
|
|
4
6
|
|
|
7
|
+
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Mixins/job/logstash-plugin-mixin-http_client-unit/)
|
|
9
|
+
|
|
5
10
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
|
6
11
|
|
|
7
12
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
|
@@ -141,4 +146,4 @@ Programming is not a required skill. Whatever you've seen about open source and
|
|
|
141
146
|
|
|
142
147
|
It is more important to the community that you are able to contribute.
|
|
143
148
|
|
|
144
|
-
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
|
149
|
+
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
|
@@ -33,11 +33,17 @@ module LogStash::PluginMixins::HttpClient
|
|
|
33
33
|
# Max number of concurrent connections to a single host. Defaults to `25`
|
|
34
34
|
config :pool_max_per_route, :validate => :number, :default => 25
|
|
35
35
|
|
|
36
|
-
# Turn this on to enable HTTP keepalive support
|
|
36
|
+
# Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
|
|
37
|
+
# one with this to fix interactions with broken keepalive implementations.
|
|
37
38
|
config :keepalive, :validate => :boolean, :default => true
|
|
38
39
|
|
|
39
|
-
# How many times should the client retry a failing URL
|
|
40
|
-
|
|
40
|
+
# How many times should the client retry a failing URL. We highly recommend NOT setting this value
|
|
41
|
+
# to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
|
|
42
|
+
# Note: if `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
|
|
43
|
+
config :automatic_retries, :validate => :number, :default => 1
|
|
44
|
+
|
|
45
|
+
# If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
|
|
46
|
+
config :retry_non_idempotent, :validate => :boolean, :default => false
|
|
41
47
|
|
|
42
48
|
# Set this to false to disable SSL/TLS certificate validation
|
|
43
49
|
# Note: setting this to false is generally considered insecure!
|
|
@@ -91,6 +97,7 @@ module LogStash::PluginMixins::HttpClient
|
|
|
91
97
|
request_timeout: @request_timeout,
|
|
92
98
|
follow_redirects: @follow_redirects,
|
|
93
99
|
automatic_retries: @automatic_retries,
|
|
100
|
+
retry_non_idempotent: @retry_non_idempotent,
|
|
94
101
|
pool_max: @pool_max,
|
|
95
102
|
pool_max_per_route: @pool_max_per_route,
|
|
96
103
|
cookies: @cookies,
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require "logstash/config/mixin"
|
|
3
|
+
|
|
4
|
+
# This module makes it easy to add a very fully configured HTTP client to logstash
|
|
5
|
+
# based on [Manticore](https://github.com/cheald/manticore).
|
|
6
|
+
# For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller
|
|
7
|
+
module LogStash::PluginMixins::HttpClient
|
|
8
|
+
class InvalidHTTPConfigError < StandardError; end
|
|
9
|
+
|
|
10
|
+
def self.included(base)
|
|
11
|
+
require 'manticore'
|
|
12
|
+
base.extend(self)
|
|
13
|
+
base.setup_http_client_config
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
public
|
|
17
|
+
def setup_http_client_config
|
|
18
|
+
# Timeout (in seconds) for the entire request
|
|
19
|
+
config :request_timeout, :validate => :number, :default => 60
|
|
20
|
+
|
|
21
|
+
# Timeout (in seconds) to wait for data on the socket. Default is `10s`
|
|
22
|
+
config :socket_timeout, :validate => :number, :default => 10
|
|
23
|
+
|
|
24
|
+
# Timeout (in seconds) to wait for a connection to be established. Default is `10s`
|
|
25
|
+
config :connect_timeout, :validate => :number, :default => 10
|
|
26
|
+
|
|
27
|
+
# Should redirects be followed? Defaults to `true`
|
|
28
|
+
config :follow_redirects, :validate => :boolean, :default => true
|
|
29
|
+
|
|
30
|
+
# Max number of concurrent connections. Defaults to `50`
|
|
31
|
+
config :pool_max, :validate => :number, :default => 50
|
|
32
|
+
|
|
33
|
+
# Max number of concurrent connections to a single host. Defaults to `25`
|
|
34
|
+
config :pool_max_per_route, :validate => :number, :default => 25
|
|
35
|
+
|
|
36
|
+
# Turn this on to enable HTTP keepalive support. We highly recommend setting `automatic_retries` to at least
|
|
37
|
+
# one with this to fix interactions with broken keepalive implementations.
|
|
38
|
+
config :keepalive, :validate => :boolean, :default => true
|
|
39
|
+
|
|
40
|
+
# How many times should the client retry a failing URL? We highly recommend NOT setting this value
|
|
41
|
+
# to zero if keepalive is enabled. Some servers incorrectly end keepalives early requiring a retry!
|
|
42
|
+
# Note, `retry_non_idempotent` is set only GET, HEAD, PUT, DELETE, OPTIONS, and TRACE requests will be retried.
|
|
43
|
+
config :automatic_retries, :validate => :number, :default => 1
|
|
44
|
+
|
|
45
|
+
# If `automatic_retries` is enabled this will cause non-idempotent HTTP verbs (such as POST) to be retried.
|
|
46
|
+
config :retry_non_idempotent, :validate => :boolean, :default => false
|
|
47
|
+
|
|
48
|
+
# Set this to false to disable SSL/TLS certificate validation
|
|
49
|
+
# Note: setting this to false is generally considered insecure!
|
|
50
|
+
config :ssl_certificate_validation, :validate => :boolean, :default => true
|
|
51
|
+
|
|
52
|
+
# If you need to use a custom X.509 CA (.pem certs) specify the path to that here
|
|
53
|
+
config :cacert, :validate => :path
|
|
54
|
+
|
|
55
|
+
# If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here
|
|
56
|
+
config :client_cert, :validate => :path
|
|
57
|
+
# If you're using a client certificate specify the path to the encryption key here
|
|
58
|
+
config :client_key, :validate => :path
|
|
59
|
+
|
|
60
|
+
# If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys!
|
|
61
|
+
config :keystore, :validate => :path
|
|
62
|
+
|
|
63
|
+
# Specify the keystore password here.
|
|
64
|
+
# Note, most .jks files created with keytool require a password!
|
|
65
|
+
config :keystore_password, :validate => :password
|
|
66
|
+
|
|
67
|
+
# Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS`
|
|
68
|
+
config :keystore_type, :validate => :string, :default => "JKS"
|
|
69
|
+
|
|
70
|
+
# If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs!
|
|
71
|
+
config :truststore, :validate => :path
|
|
72
|
+
|
|
73
|
+
# Specify the truststore password here.
|
|
74
|
+
# Note, most .jks files created with keytool require a password!
|
|
75
|
+
config :truststore_password, :validate => :password
|
|
76
|
+
|
|
77
|
+
# Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
|
|
78
|
+
config :truststore_type, :validate => :string, :default => "JKS"
|
|
79
|
+
|
|
80
|
+
# Enable cookie support. With this enabled the client will persist cookies
|
|
81
|
+
# across requests as a normal web browser would. Enabled by default
|
|
82
|
+
config :cookies, :validate => :boolean, :default => true
|
|
83
|
+
|
|
84
|
+
# If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
|
|
85
|
+
#
|
|
86
|
+
# 1. Proxy host in form: `http://proxy.org:1234`
|
|
87
|
+
# 2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
|
|
88
|
+
# 3. Proxy host in form: `{url => 'http://proxy.org:1234', user => 'username@host', password => 'password'}`
|
|
89
|
+
config :proxy
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
public
|
|
93
|
+
def client_config
|
|
94
|
+
c = {
|
|
95
|
+
connect_timeout: @connect_timeout,
|
|
96
|
+
socket_timeout: @socket_timeout,
|
|
97
|
+
request_timeout: @request_timeout,
|
|
98
|
+
follow_redirects: @follow_redirects,
|
|
99
|
+
automatic_retries: @automatic_retries,
|
|
100
|
+
retry_non_idempotent: @retry_non_idempotent,
|
|
101
|
+
pool_max: @pool_max,
|
|
102
|
+
pool_max_per_route: @pool_max_per_route,
|
|
103
|
+
cookies: @cookies,
|
|
104
|
+
keepalive: @keepalive,
|
|
105
|
+
verify: @ssl_certificate_validation
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if @proxy
|
|
109
|
+
# Symbolize keys if necessary
|
|
110
|
+
c[:proxy] = @proxy.is_a?(Hash) ?
|
|
111
|
+
@proxy.reduce({}) {|memo,(k,v)| memo[k.to_sym] = v; memo} :
|
|
112
|
+
@proxy
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
c[:ssl] = {}
|
|
116
|
+
if @cacert
|
|
117
|
+
c[:ssl][:ca_file] = @cacert
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
if @truststore
|
|
121
|
+
c[:ssl].merge!(
|
|
122
|
+
:truststore => @truststore,
|
|
123
|
+
:truststore_type => @truststore_type
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
# JKS files have optional passwords if programatically created
|
|
127
|
+
if (@truststore_password)
|
|
128
|
+
c[:ssl].merge!(truststore_password: @truststore_password.value)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
if @keystore
|
|
133
|
+
c[:ssl].merge!(
|
|
134
|
+
:keystore => @keystore,
|
|
135
|
+
:keystore_type => @keystore_type
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# JKS files have optional passwords if programatically created
|
|
139
|
+
if keystore_password
|
|
140
|
+
c[:ssl].merge!(keystore_password: @keystore_password.value)
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
if @client_cert && @client_key
|
|
145
|
+
c[:ssl][:client_cert] = @client_cert
|
|
146
|
+
c[:ssl][:client_key] = @client_key
|
|
147
|
+
elsif !!@client_cert ^ !!@client_key
|
|
148
|
+
raise InvalidHTTPConfigError, "You must specify both client_cert and client_key for an HTTP client, or neither!"
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
c
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
private
|
|
155
|
+
def make_client
|
|
156
|
+
Manticore::Client.new(client_config)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
public
|
|
160
|
+
def client
|
|
161
|
+
@client ||= make_client
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-mixin-http_client'
|
|
3
|
-
s.version = '2.0
|
|
3
|
+
s.version = '2.1.0'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = "AWS mixins to provide a unified interface for Amazon Webservice"
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-mixin-http_client
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0
|
|
4
|
+
version: 2.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-
|
|
11
|
+
date: 2015-12-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
15
15
|
requirements:
|
|
16
|
-
- -
|
|
16
|
+
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
18
|
version: 2.0.0.beta2
|
|
19
|
-
- - <
|
|
19
|
+
- - "<"
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
21
|
version: 3.0.0
|
|
22
22
|
name: logstash-core
|
|
@@ -24,16 +24,16 @@ dependencies:
|
|
|
24
24
|
type: :runtime
|
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
26
26
|
requirements:
|
|
27
|
-
- -
|
|
27
|
+
- - ">="
|
|
28
28
|
- !ruby/object:Gem::Version
|
|
29
29
|
version: 2.0.0.beta2
|
|
30
|
-
- - <
|
|
30
|
+
- - "<"
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
32
|
version: 3.0.0
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
|
35
35
|
requirements:
|
|
36
|
-
- -
|
|
36
|
+
- - ">="
|
|
37
37
|
- !ruby/object:Gem::Version
|
|
38
38
|
version: '0'
|
|
39
39
|
name: logstash-codec-plain
|
|
@@ -41,13 +41,13 @@ dependencies:
|
|
|
41
41
|
type: :runtime
|
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
|
43
43
|
requirements:
|
|
44
|
-
- -
|
|
44
|
+
- - ">="
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
46
|
version: '0'
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
|
49
49
|
requirements:
|
|
50
|
-
- -
|
|
50
|
+
- - ">="
|
|
51
51
|
- !ruby/object:Gem::Version
|
|
52
52
|
version: 0.4.1
|
|
53
53
|
name: manticore
|
|
@@ -55,13 +55,13 @@ dependencies:
|
|
|
55
55
|
type: :runtime
|
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
|
57
57
|
requirements:
|
|
58
|
-
- -
|
|
58
|
+
- - ">="
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
60
|
version: 0.4.1
|
|
61
61
|
- !ruby/object:Gem::Dependency
|
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
|
63
63
|
requirements:
|
|
64
|
-
- -
|
|
64
|
+
- - ">="
|
|
65
65
|
- !ruby/object:Gem::Version
|
|
66
66
|
version: '0'
|
|
67
67
|
name: logstash-devutils
|
|
@@ -69,13 +69,13 @@ dependencies:
|
|
|
69
69
|
type: :development
|
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
|
72
|
-
- -
|
|
72
|
+
- - ">="
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
74
|
version: '0'
|
|
75
75
|
- !ruby/object:Gem::Dependency
|
|
76
76
|
requirement: !ruby/object:Gem::Requirement
|
|
77
77
|
requirements:
|
|
78
|
-
- -
|
|
78
|
+
- - ">="
|
|
79
79
|
- !ruby/object:Gem::Version
|
|
80
80
|
version: '0'
|
|
81
81
|
name: stud
|
|
@@ -83,7 +83,7 @@ dependencies:
|
|
|
83
83
|
type: :development
|
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
|
85
85
|
requirements:
|
|
86
|
-
- -
|
|
86
|
+
- - ">="
|
|
87
87
|
- !ruby/object:Gem::Version
|
|
88
88
|
version: '0'
|
|
89
89
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
|
@@ -98,6 +98,7 @@ files:
|
|
|
98
98
|
- LICENSE
|
|
99
99
|
- README.md
|
|
100
100
|
- lib/logstash/plugin_mixins/http_client.rb
|
|
101
|
+
- lib/logstash/plugin_mixins/http_client.rb~
|
|
101
102
|
- logstash-mixin-http_client.gemspec
|
|
102
103
|
- spec/plugin_mixin/http_client_spec.rb
|
|
103
104
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
|
@@ -110,12 +111,12 @@ require_paths:
|
|
|
110
111
|
- lib
|
|
111
112
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
112
113
|
requirements:
|
|
113
|
-
- -
|
|
114
|
+
- - ">="
|
|
114
115
|
- !ruby/object:Gem::Version
|
|
115
116
|
version: '0'
|
|
116
117
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
117
118
|
requirements:
|
|
118
|
-
- -
|
|
119
|
+
- - ">="
|
|
119
120
|
- !ruby/object:Gem::Version
|
|
120
121
|
version: '0'
|
|
121
122
|
requirements: []
|