logstash-integration-logstash 0.0.5-java → 1.0.1-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/VERSION +1 -1
- data/docs/index.asciidoc +8 -5
- data/docs/output-logstash.asciidoc +6 -6
- data/lib/logstash/outputs/logstash.rb +254 -127
- data/lib/logstash/utils/load_balancer.rb +81 -0
- data/logstash-integration-logstash.gemspec +2 -1
- data/spec/fixtures/certs/generated/client_from_root.jks +0 -0
- data/spec/fixtures/certs/generated/client_from_root.key.pem +50 -50
- data/spec/fixtures/certs/generated/client_from_root.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/client_from_root.p12 +0 -0
- data/spec/fixtures/certs/generated/client_from_root.pem +28 -28
- data/spec/fixtures/certs/generated/client_from_untrusted.jks +0 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.key.pem +50 -50
- data/spec/fixtures/certs/generated/client_from_untrusted.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/client_from_untrusted.p12 +0 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.pem +28 -28
- data/spec/fixtures/certs/generated/client_self_signed.jks +0 -0
- data/spec/fixtures/certs/generated/client_self_signed.key.pem +50 -50
- data/spec/fixtures/certs/generated/client_self_signed.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/client_self_signed.p12 +0 -0
- data/spec/fixtures/certs/generated/client_self_signed.pem +28 -28
- data/spec/fixtures/certs/generated/root.key.pem +50 -50
- data/spec/fixtures/certs/generated/root.pem +28 -28
- data/spec/fixtures/certs/generated/server_from_root-key-pkcs8.pem +50 -50
- data/spec/fixtures/certs/generated/server_from_root.jks +0 -0
- data/spec/fixtures/certs/generated/server_from_root.key.pem +50 -50
- data/spec/fixtures/certs/generated/server_from_root.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/server_from_root.p12 +0 -0
- data/spec/fixtures/certs/generated/server_from_root.pem +28 -28
- data/spec/fixtures/certs/generated/untrusted.key.pem +50 -50
- data/spec/fixtures/certs/generated/untrusted.pem +28 -28
- data/spec/unit/full_transmission_spec.rb +10 -2
- data/spec/unit/load_balancer_spec.rb +67 -0
- data/spec/unit/logstash_output_spec.rb +179 -17
- metadata +22 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c37aa756e32d9800664488bd5fb288fba500d3556a9a9b882ec4e8bf6ebb5cac
|
4
|
+
data.tar.gz: c1b7a79793a56572bfac9c11114d52dbe9b7a5f86a120d34db28282cdec02aaf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 70a76600466c636bc8d8c87618760fae7a34f81ead194e812d41166e795bdd5df60cc42be4d8e875e9bf2631d0c5b99d2511c5fd356056821d1318470929a120
|
7
|
+
data.tar.gz: 6465e411376f71cf53270a37e0127aa21ea415d481f332dd706e9f5c654ef86523f341d00cacd538b125840751a8dd3ab7e141b344ae6b8b46a13e49a072517d
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 1.0.1
|
2
|
+
- Fix: improves throughput by allowing pipeline workers to share a plugin instance _concurrently_ instead of _sequentially_ [#19](https://github.com/logstash-plugins/logstash-integration-logstash/pull/19)
|
3
|
+
|
4
|
+
## 1.0.0
|
5
|
+
- Introduces the load balancing mechanism to distribute the requests among the `hosts` [#16](https://github.com/logstash-plugins/logstash-integration-logstash/pull/16)
|
6
|
+
|
1
7
|
## 0.0.5
|
2
8
|
- [DOC] Fixes to link formatting [#15](https://github.com/logstash-plugins/logstash-integration-logstash/pull/15)
|
3
9
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
1.0.1
|
data/docs/index.asciidoc
CHANGED
@@ -21,7 +21,7 @@ include::{include_path}/plugin_header.asciidoc[]
|
|
21
21
|
|
22
22
|
==== Description
|
23
23
|
|
24
|
-
The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another:
|
24
|
+
The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another instance(s):
|
25
25
|
|
26
26
|
* {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin]
|
27
27
|
* {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin]
|
@@ -29,7 +29,7 @@ The Logstash Integration Plugin provides integrated plugins for sending events f
|
|
29
29
|
[id="plugins-{type}s-{plugin}-concepts"]
|
30
30
|
===== High-level concepts
|
31
31
|
|
32
|
-
You can configure a `logstash` output to send events to
|
32
|
+
You can configure a `logstash` output to send events to one or more `logstash` inputs, which are each in another pipeline that is running in different processes or on a different host.
|
33
33
|
|
34
34
|
To do so, you should first configure the downstream pipeline with a `logstash` input plugin, bound to an available port so that it can listen for inbound connections.
|
35
35
|
Security is enabled by default, so you will need to either provide identity material or disable SSL.
|
@@ -53,13 +53,11 @@ input {
|
|
53
53
|
Once the downstream pipeline is configured and running, you may send events from any number of upstream pipelines by adding a `logstash` output plugin that points to the downstream input.
|
54
54
|
You may need to configure SSL to trust the certificates presented by the downstream input plugin.
|
55
55
|
|
56
|
-
NOTE: Single host endpoint is supported for `hosts`. Multi-host support is coming soon.
|
57
|
-
|
58
56
|
[source]
|
59
57
|
----
|
60
58
|
output {
|
61
59
|
logstash {
|
62
|
-
hosts => "10.0.0.123:9800"
|
60
|
+
hosts => ["10.0.0.123:9800", "10.0.0.125:9801"]
|
63
61
|
|
64
62
|
# SSL TRUST <1>
|
65
63
|
ssl_truststore_path => "/path/to/truststore.p12"
|
@@ -69,4 +67,9 @@ output {
|
|
69
67
|
----
|
70
68
|
<1> Unless SSL is disabled or the downstream input is expected to present certificates signed by globally-trusted authorities, you will likely need to provide a source-of-trust.
|
71
69
|
|
70
|
+
[id="plugins-{type}s-{plugin}-load-balancing"]
|
71
|
+
==== Load Balancing
|
72
|
+
|
73
|
+
When a `logstash` output is configured to send to multiple `hosts`, it distributes events in batches to _all_ of those downstream hosts fairly, favoring those without recent errors. This increases the likelihood of each batch being routed to a downstream that is up and has capacity to receive events.
|
74
|
+
|
72
75
|
:no_codec!:
|
@@ -103,7 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
103
103
|
[cols="<,<,<",options="header",]
|
104
104
|
|=======================================================================
|
105
105
|
|Setting |Input type |Required
|
106
|
-
| <<plugins-{type}s-{plugin}-hosts>>
|
106
|
+
| <<plugins-{type}s-{plugin}-hosts>> |list of <<string,string>> |Yes
|
107
107
|
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
108
108
|
| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
|
109
109
|
| <<plugins-{type}s-{plugin}-ssl_certificate>> | <<path,path>>|No
|
@@ -125,27 +125,27 @@ output plugins.
|
|
125
125
|
[id="plugins-{type}s-{plugin}-hosts"]
|
126
126
|
===== `hosts`
|
127
127
|
|
128
|
-
* Value type is <<string,string>>
|
128
|
+
* Value type is list of <<string,string>>
|
129
129
|
* There is no default value for this setting.
|
130
130
|
* Constraints:
|
131
131
|
** When using IPv6, IP address must be in an enclosed in brackets.
|
132
132
|
** When a port is not provided, the default `9800` is used.
|
133
133
|
|
134
|
-
|
135
|
-
|
136
|
-
NOTE: Single host endpoint is supported for `hosts`. Multi-host support is coming soon.
|
134
|
+
The addresses of one or more downstream `input`s to connect to.
|
137
135
|
|
138
136
|
Host can be any of IPv4, IPv6 (in enclosed bracket) or host name, examples:
|
139
137
|
|
140
138
|
* `"127.0.0.1"`
|
141
139
|
* `"127.0.0.1:9801"`
|
142
140
|
* `"ds.example.com"`
|
143
|
-
* `"ds.example:9802"`
|
141
|
+
* `"ds.example.com:9802"`
|
144
142
|
* `"[::1]"`
|
145
143
|
* `"[::1]:9803"`
|
146
144
|
* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"`
|
147
145
|
* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9804"`
|
148
146
|
|
147
|
+
Plugin balances incoming load among the `hosts`. For more information, visit {logstash-ref}/plugins-integrations-logstash.html[Logstash integration plugin] _Load Balancing_ section.
|
148
|
+
|
149
149
|
When connecting, communication to downstream input {ls} is secured with SSL unless configured otherwise.
|
150
150
|
|
151
151
|
[WARNING]
|
@@ -1,58 +1,62 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'logstash/outputs/base'
|
4
|
-
require 'logstash/namespace'
|
5
3
|
|
6
|
-
require "logstash/
|
4
|
+
require "logstash/outputs/base"
|
5
|
+
require "logstash/namespace"
|
6
|
+
|
7
|
+
require 'logstash/plugin_mixins/normalize_config_support'
|
8
|
+
require "logstash/plugin_mixins/http_client"
|
7
9
|
require "logstash/plugin_mixins/validator_support/required_host_optional_port_validation_adapter"
|
10
|
+
require "zlib"
|
11
|
+
|
12
|
+
require "stud/interval" # Stud::stoppable_sleep
|
8
13
|
|
9
14
|
class LogStash::Outputs::Logstash < LogStash::Outputs::Base
|
15
|
+
|
16
|
+
concurrency :shared
|
17
|
+
|
10
18
|
extend LogStash::PluginMixins::ValidatorSupport::RequiredHostOptionalPortValidationAdapter
|
11
19
|
|
12
|
-
include LogStash::PluginMixins::
|
20
|
+
include LogStash::PluginMixins::HttpClient[:with_deprecated => false]
|
21
|
+
include LogStash::PluginMixins::NormalizeConfigSupport
|
22
|
+
|
23
|
+
require "logstash/utils/load_balancer"
|
13
24
|
|
14
25
|
config_name "logstash"
|
15
26
|
|
16
27
|
# Sets the host of the downstream Logstash instance.
|
17
28
|
# Host can be any of IPv4, IPv6 (requires to be in enclosed bracket) or host name, the forms:
|
18
|
-
# `"127.0.0.1"`
|
19
|
-
# `"127.0.0.1:
|
20
|
-
# `"foo-bar.com"`
|
21
|
-
# `"
|
22
|
-
# `"[::1]"`
|
23
|
-
# `"[::1]:9000"`
|
29
|
+
# `"127.0.0.1"` or `["127.0.0.1"]` if single host with default port
|
30
|
+
# `"127.0.0.1:9801"` or `["127.0.0.1:9801"]` if single host with custom port
|
31
|
+
# `["foo-bar.com", "foo-bar.com:9800"]`
|
32
|
+
# `["[::1]", "[::1]:9000"]`
|
24
33
|
# `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"`
|
25
34
|
#
|
26
|
-
|
27
|
-
#
|
28
|
-
config :hosts, :validate => :required_host_optional_port, :required => true
|
35
|
+
config :hosts, :validate => :required_host_optional_port, :list => true, :required => true
|
29
36
|
|
30
|
-
|
31
|
-
config :username, :validate => :string, :required => false
|
32
|
-
config :password, :validate => :password, :required => false
|
37
|
+
config :username, :validate => :string, :required => false
|
33
38
|
|
34
|
-
config :ssl_enabled,
|
39
|
+
config :ssl_enabled, :validate => :boolean, :default => true
|
35
40
|
|
36
|
-
|
37
|
-
config :ssl_certificate, :validate => :path
|
38
|
-
config :ssl_key, :validate => :path
|
41
|
+
config :user, :validate => :string, :deprecated => "Use `username` instead.", :required => false
|
39
42
|
|
40
|
-
|
41
|
-
config :ssl_keystore_path, :validate => :path
|
42
|
-
config :ssl_keystore_password, :validate => :password
|
43
|
-
|
44
|
-
# SSL:TRUST:CONFIG
|
45
|
-
config :ssl_verification_mode, :validate => %w(full none), :default => 'full'
|
46
|
-
|
47
|
-
# SSL:TRUST:SOURCE ca file
|
48
|
-
config :ssl_certificate_authorities, :validate => :path, :list => true
|
49
|
-
|
50
|
-
# SSL:TRUST:SOURCE truststore
|
51
|
-
config :ssl_truststore_path, :validate => :path
|
52
|
-
config :ssl_truststore_password, :validate => :password
|
43
|
+
DEFAULT_PORT = 9800.freeze
|
53
44
|
|
54
|
-
|
55
|
-
|
45
|
+
RETRIABLE_CODES = [429, 500, 502, 503, 504]
|
46
|
+
RETRYABLE_MANTICORE_EXCEPTIONS = [
|
47
|
+
::Manticore::Timeout,
|
48
|
+
::Manticore::SocketException,
|
49
|
+
::Manticore::ClientProtocolException,
|
50
|
+
::Manticore::ResolutionFailure,
|
51
|
+
::Manticore::SocketTimeout
|
52
|
+
]
|
53
|
+
RETRYABLE_EXCEPTION_PATTERN = Regexp.union([
|
54
|
+
/Connection reset by peer/i,
|
55
|
+
/Read Timed out/i,
|
56
|
+
])
|
57
|
+
|
58
|
+
# @api private
|
59
|
+
attr_reader :http_client
|
56
60
|
|
57
61
|
def initialize(*a)
|
58
62
|
super
|
@@ -61,129 +65,252 @@ class LogStash::Outputs::Logstash < LogStash::Outputs::Base
|
|
61
65
|
fail LogStash::ConfigurationError, 'The `logstash` output does not have an externally-configurable `codec`'
|
62
66
|
end
|
63
67
|
|
64
|
-
|
65
|
-
|
66
|
-
|
68
|
+
@headers = {
|
69
|
+
"Content-Type" => "application/x-ndjson".freeze,
|
70
|
+
"Content-Encoding" => "gzip".freeze
|
71
|
+
}.freeze
|
67
72
|
|
68
|
-
logger.debug("
|
69
|
-
@internal_http = plugin_factory.output('http').new(inner_http_output_options)
|
70
|
-
logger.debug("inner HTTP output plugin has been initialized")
|
73
|
+
logger.debug("`logstash` output plugin has been initialized")
|
71
74
|
end
|
72
75
|
|
73
76
|
def register
|
74
|
-
logger.debug("
|
75
|
-
|
76
|
-
|
77
|
+
logger.debug("Registering `logstash` output plugin")
|
78
|
+
|
79
|
+
@username = normalize_config(:username) do |normalize|
|
80
|
+
normalize.with_deprecated_alias(:user)
|
81
|
+
end
|
82
|
+
# remove after deprecating user in the http-mixin
|
83
|
+
@user = @username ? @username.freeze : @user
|
84
|
+
|
85
|
+
validate_auth_settings!
|
86
|
+
|
87
|
+
if @ssl_enabled == false
|
88
|
+
rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
|
89
|
+
fail(LogStash::ConfigurationError, "Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
|
90
|
+
end
|
91
|
+
|
92
|
+
validate_ssl_identity_options!
|
93
|
+
validate_ssl_trust_options!
|
94
|
+
|
95
|
+
# if we don't initialize now, we get runtime error when sending events if there are issues with configs
|
96
|
+
@http_client = client
|
97
|
+
fail(LogStash::ConfigurationError, "`hosts` must not be empty") if @hosts.empty?
|
98
|
+
|
99
|
+
@load_balancer = LoadBalancer.new(normalize_host_uris)
|
100
|
+
|
101
|
+
logger.debug("`logstash` output plugin has been registered")
|
102
|
+
end
|
103
|
+
|
104
|
+
def validate_auth_settings!
|
105
|
+
if @username
|
106
|
+
fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided') if @password.nil?
|
107
|
+
logger.warn("Transmitting credentials over non-secured connection") if @ssl_enabled == false
|
108
|
+
elsif @password
|
109
|
+
fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def validate_ssl_identity_options!
|
114
|
+
if @ssl_certificate && @ssl_keystore_path
|
115
|
+
fail(LogStash::ConfigurationError, "SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both")
|
116
|
+
elsif @ssl_certificate
|
117
|
+
fail(LogStash::ConfigurationError, "`ssl_key` is REQUIRED when `ssl_certificate` is provided") if @ssl_key.nil?
|
118
|
+
elsif @ssl_key
|
119
|
+
fail(LogStash::ConfigurationError, "`ssl_key` is not allowed unless `ssl_certificate` is configured")
|
120
|
+
elsif @ssl_keystore_path
|
121
|
+
fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided") if @ssl_keystore_password.nil?
|
122
|
+
elsif @ssl_keystore_password
|
123
|
+
fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
|
124
|
+
else
|
125
|
+
# acceptable
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def validate_ssl_trust_options!
|
130
|
+
if @ssl_certificate_authorities&.any? && @ssl_truststore_path
|
131
|
+
fail(LogStash::ConfigurationError, "SSL trust can be configured with EITHER `ssl_certificate_authorities` OR `ssl_truststore_*`, but not both")
|
132
|
+
elsif @ssl_certificate_authorities&.any?
|
133
|
+
fail(LogStash::ConfigurationError, "SSL Certificate Authorities cannot be configured when `ssl_verification_mode => none`") if @ssl_verification_mode == 'none'
|
134
|
+
elsif @ssl_truststore_path
|
135
|
+
fail(LogStash::ConfigurationError, "SSL Truststore cannot be configured when `ssl_verification_mode => none`") if @ssl_verification_mode == 'none'
|
136
|
+
fail(LogStash::ConfigurationError, "`ssl_truststore_password` is REQUIRED when `ssl_truststore_path` is provided") if @ssl_truststore_password.nil?
|
137
|
+
elsif @ssl_truststore_password
|
138
|
+
fail(LogStash::ConfigurationError, "`ssl_truststore_password` not allowed unless `ssl_truststore_path` is configured")
|
139
|
+
end
|
77
140
|
end
|
78
141
|
|
79
142
|
def multi_receive(events)
|
80
143
|
return if events.empty?
|
81
|
-
|
82
|
-
|
83
|
-
rescue => e
|
84
|
-
logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
|
85
|
-
raise
|
144
|
+
|
145
|
+
send_events(events)
|
86
146
|
end
|
87
147
|
|
88
148
|
def stop
|
89
|
-
logger.debug("
|
90
|
-
@internal_http.stop
|
91
|
-
logger.debug('inner HTTP output plugin has been stopped')
|
149
|
+
logger.debug("`logstash` output plugin has been stopped")
|
92
150
|
end
|
93
151
|
|
94
152
|
def close
|
95
|
-
logger.debug("
|
96
|
-
|
97
|
-
logger.debug(
|
153
|
+
logger.debug("Closing `logstash` output plugin")
|
154
|
+
http_client.close
|
155
|
+
logger.debug("`logstash` output plugin has been closed")
|
98
156
|
end
|
99
157
|
|
100
|
-
|
158
|
+
private
|
101
159
|
|
102
|
-
def
|
103
|
-
@
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
'http_compression' => true,
|
114
|
-
}
|
115
|
-
|
116
|
-
if @username
|
117
|
-
http_options['user'] = @username
|
118
|
-
http_options['password'] = @password || fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided')
|
119
|
-
logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
|
120
|
-
elsif @password
|
121
|
-
fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
|
122
|
-
end
|
160
|
+
def normalize_host_uris
|
161
|
+
@_normalized_host_uris ||= begin
|
162
|
+
scheme = @ssl_enabled ? 'https' : 'http'
|
163
|
+
@hosts.map do |destination| # Struct(:host,:port)
|
164
|
+
URI::Generic.build(:scheme => scheme,
|
165
|
+
:host => destination.host,
|
166
|
+
:port => destination.port || DEFAULT_PORT)
|
167
|
+
end.map(&:to_s).map(&:freeze)
|
168
|
+
end
|
169
|
+
end
|
123
170
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
else
|
128
|
-
http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
|
171
|
+
def send_events(events)
|
172
|
+
body = LogStash::Json.dump(events.map(&:to_hash))
|
173
|
+
compressed_body = gzip(body)
|
129
174
|
|
130
|
-
|
131
|
-
|
132
|
-
end
|
175
|
+
next_backoff = 0.1
|
176
|
+
max_backoff = 30
|
133
177
|
|
134
|
-
|
178
|
+
loop do
|
179
|
+
next_action = transmit(body, compressed_body)
|
180
|
+
break unless next_action == :retry
|
181
|
+
|
182
|
+
Stud.stoppable_sleep(next_backoff) { pipeline_shutdown_requested? }
|
183
|
+
next_backoff = [next_backoff*2, max_backoff].min
|
184
|
+
|
185
|
+
if pipeline_shutdown_requested?
|
186
|
+
logger.warn "Aborting the batch due to shutdown request"
|
187
|
+
abort_batch_if_available!
|
188
|
+
break # legacy abort (lossy)
|
189
|
+
end
|
135
190
|
end
|
191
|
+
rescue => e
|
192
|
+
# This should never happen unless there's a flat out bug in the code
|
193
|
+
logger.error("Error occurred while sending events",
|
194
|
+
:class => e.class.name,
|
195
|
+
:message => e.message,
|
196
|
+
:backtrace => e.backtrace)
|
197
|
+
raise e
|
136
198
|
end
|
137
199
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
200
|
+
# The exceptions we decide (retriable or abort) to let the Load Balancer know
|
201
|
+
TransmitException = Class.new(RuntimeError)
|
202
|
+
private_constant :TransmitException
|
203
|
+
|
204
|
+
RetriableTransmitException = Class.new(TransmitException)
|
205
|
+
private_constant :RetriableTransmitException
|
206
|
+
|
207
|
+
TerminalTransmitException = Class.new(TransmitException)
|
208
|
+
private_constant :TerminalTransmitException
|
209
|
+
|
210
|
+
##
|
211
|
+
# @param body [String]
|
212
|
+
# @param compressed_body [String]
|
213
|
+
# @return [:done, :abort, :retry]
|
214
|
+
def transmit(body, compressed_body)
|
215
|
+
@load_balancer.select do |selected_host_uri|
|
216
|
+
response = begin
|
217
|
+
http_client.post(selected_host_uri, :body => compressed_body, :headers => @headers).call
|
218
|
+
rescue => exception
|
219
|
+
retryable_exception = retryable_exception?(exception)
|
220
|
+
log_exception(selected_host_uri, exception, body, retryable_exception)
|
221
|
+
|
222
|
+
# raise exception to mar the host error
|
223
|
+
raise retryable_exception ? RetriableTransmitException : TerminalTransmitException
|
224
|
+
end
|
225
|
+
|
226
|
+
return :done if response_success?(response.code)
|
227
|
+
|
228
|
+
retryable_response = retryable_response?(response.code)
|
229
|
+
log_response(selected_host_uri, response, body, retryable_response)
|
230
|
+
|
231
|
+
# raise exception to mar the host error
|
232
|
+
raise retryable_response ? RetriableTransmitException : TerminalTransmitException
|
233
|
+
end
|
234
|
+
rescue RetriableTransmitException => exception
|
235
|
+
return :retry
|
236
|
+
rescue TerminalTransmitException => exception
|
237
|
+
return :abort
|
145
238
|
end
|
146
239
|
|
147
|
-
def
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
elsif @ssl_key
|
156
|
-
fail(LogStash::ConfigurationError, '`ssl_key` is not allowed unless `ssl_certificate` is configured')
|
157
|
-
elsif @ssl_keystore_path
|
158
|
-
return {
|
159
|
-
'ssl_keystore_path' => @ssl_keystore_path,
|
160
|
-
'ssl_keystore_password' => @ssl_keystore_password || fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided"),
|
161
|
-
}
|
162
|
-
elsif @ssl_keystore_password
|
163
|
-
fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
|
240
|
+
def log_response(uri, response, body, retriable)
|
241
|
+
response_code = response.code
|
242
|
+
if retriable
|
243
|
+
if response_code == 429
|
244
|
+
logger.debug("Encountered a retriable 429 response")
|
245
|
+
else
|
246
|
+
logger.warn("Encountered a retryable error in `logstash` output", :code => response_code, :body => response.body)
|
247
|
+
end
|
164
248
|
else
|
165
|
-
|
249
|
+
logger.error("Encountered error",
|
250
|
+
:response_code => response_code,
|
251
|
+
:host => uri,
|
252
|
+
:body => body
|
253
|
+
)
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
def log_exception(uri, exception, body, retriable)
|
258
|
+
log_entry = { :host => uri, :message => exception.message, :class => exception.class, :retry => retriable }
|
259
|
+
if logger.debug?
|
260
|
+
# backtraces are big
|
261
|
+
log_entry[:backtrace] = exception.backtrace
|
262
|
+
# body can be big and may have sensitive data
|
263
|
+
log_entry[:body] = body
|
166
264
|
end
|
265
|
+
logger.error("Could not send data to host", log_entry)
|
167
266
|
end
|
168
267
|
|
169
|
-
def
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
268
|
+
def gzip(data)
|
269
|
+
gz = StringIO.new
|
270
|
+
gz.set_encoding("BINARY")
|
271
|
+
z = Zlib::GzipWriter.new(gz)
|
272
|
+
z.write(data)
|
273
|
+
z.close
|
274
|
+
gz.string
|
275
|
+
end
|
177
276
|
|
178
|
-
|
179
|
-
|
180
|
-
|
277
|
+
def response_success?(response_code)
|
278
|
+
response_code >= 200 && response_code <= 299
|
279
|
+
end
|
181
280
|
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
281
|
+
def retryable_exception?(exception)
|
282
|
+
retryable_manticore_exception?(exception) || retryable_unknown_exception?(exception)
|
283
|
+
end
|
284
|
+
|
285
|
+
def retryable_manticore_exception?(exception)
|
286
|
+
RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me)}
|
287
|
+
end
|
288
|
+
|
289
|
+
def retryable_unknown_exception?(exception)
|
290
|
+
exception.is_a?(::Manticore::UnknownException) &&
|
291
|
+
RETRYABLE_EXCEPTION_PATTERN.match?(exception.message)
|
292
|
+
end
|
293
|
+
|
294
|
+
def retryable_response?(response_code)
|
295
|
+
RETRIABLE_CODES.include?(response_code)
|
296
|
+
end
|
297
|
+
|
298
|
+
# Emulate `pipeline_shutdown_requested?` when running on older Logstash
|
299
|
+
unless ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.1.0')
|
300
|
+
def pipeline_shutdown_requested?
|
301
|
+
execution_context&.pipeline&.shutdown_requested?
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
# When running on Logstash that can abort batches,
|
306
|
+
# raise the required exception, do nothing otherwise.
|
307
|
+
if ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.8.0')
|
308
|
+
def abort_batch_if_available!
|
309
|
+
raise org.logstash.execution.AbortedBatchException.new
|
310
|
+
end
|
311
|
+
else
|
312
|
+
def abort_batch_if_available!
|
313
|
+
nil
|
187
314
|
end
|
188
315
|
end
|
189
316
|
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
|
4
|
+
require "monitor"
|
5
|
+
|
6
|
+
class LoadBalancer
|
7
|
+
include MonitorMixin
|
8
|
+
|
9
|
+
##
|
10
|
+
# Creates a new Router with the provided downstream_infos
|
11
|
+
# that ignores errors older than the cool_off period
|
12
|
+
# @param host_infos [Enumerable<HostSate>]: a list of downstream hosts
|
13
|
+
# to include in routing
|
14
|
+
# @param cool_off [Integer]: The cool_off period in seconds in which downstreams with
|
15
|
+
# recent errors are de-prioritized (default: 60)
|
16
|
+
def initialize(host_infos, cool_off: 60)
|
17
|
+
super() # to initialize MonitorMixin
|
18
|
+
|
19
|
+
fail ArgumentError, "Non-empty `host_infos` hosts required." unless host_infos&.any?
|
20
|
+
fail ArgumentError, "`cool_off` requires integer value." unless cool_off.kind_of?(Integer)
|
21
|
+
|
22
|
+
@cool_off = cool_off
|
23
|
+
@host_states = host_infos.map do |host_info|
|
24
|
+
HostState.new(host_info)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
##
|
29
|
+
# Yields the block with a {HostState}, prioritizing
|
30
|
+
# hosts that are less concurrently-used and which have
|
31
|
+
# not errored recently.
|
32
|
+
# @yield param selected [HostState]
|
33
|
+
def select
|
34
|
+
selected = synchronize { pick_one.tap(&:increment) }
|
35
|
+
yield selected.uri
|
36
|
+
rescue
|
37
|
+
synchronize { selected.mark_error }
|
38
|
+
raise
|
39
|
+
ensure
|
40
|
+
synchronize { selected.decrement }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def pick_one
|
46
|
+
threshold = Time.now.to_i - @cool_off
|
47
|
+
@host_states.sort_by do |host_state|
|
48
|
+
[
|
49
|
+
[host_state.last_error, threshold].max, # deprioritize recent errors
|
50
|
+
host_state.concurrent, # deprioritize high concurrency
|
51
|
+
host_state.last_start # deprioritize recent use
|
52
|
+
]
|
53
|
+
end.first
|
54
|
+
end
|
55
|
+
|
56
|
+
class HostState
|
57
|
+
def initialize(host_uri)
|
58
|
+
@uri = host_uri
|
59
|
+
@last_error = 0
|
60
|
+
@concurrent = 0
|
61
|
+
@last_start = 0
|
62
|
+
end
|
63
|
+
attr_reader :uri
|
64
|
+
attr_reader :last_error
|
65
|
+
attr_reader :concurrent
|
66
|
+
attr_reader :last_start
|
67
|
+
|
68
|
+
def increment
|
69
|
+
@concurrent += 1
|
70
|
+
@last_start = Time.now.to_f
|
71
|
+
end
|
72
|
+
|
73
|
+
def decrement
|
74
|
+
@concurrent -= 1
|
75
|
+
end
|
76
|
+
|
77
|
+
def mark_error
|
78
|
+
@last_error = Time.now.to_i
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -28,8 +28,9 @@ Gem::Specification.new do |s|
|
|
28
28
|
s.add_runtime_dependency "logstash-mixin-validator_support", "~> 1.1"
|
29
29
|
s.add_runtime_dependency "logstash-codec-json_lines", "~> 3.1"
|
30
30
|
|
31
|
+
s.add_runtime_dependency "logstash-mixin-http_client", "~> 7.3"
|
31
32
|
s.add_runtime_dependency "logstash-input-http", ">= 3.7.0" # some params not available in older versions because they are renamed, such as `cacert` to `ssl_certificate_authorities`
|
32
|
-
s.add_runtime_dependency "
|
33
|
+
s.add_runtime_dependency "stud"
|
33
34
|
|
34
35
|
s.add_development_dependency "logstash-devutils"
|
35
36
|
s.add_development_dependency "rspec-collection_matchers"
|
Binary file
|