logstash-integration-logstash 0.0.5-java → 1.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/VERSION +1 -1
- data/docs/index.asciidoc +8 -5
- data/docs/output-logstash.asciidoc +6 -6
- data/lib/logstash/outputs/logstash.rb +251 -127
- data/lib/logstash/utils/load_balancer.rb +81 -0
- data/logstash-integration-logstash.gemspec +2 -1
- data/spec/fixtures/certs/generated/client_from_root.jks +0 -0
- data/spec/fixtures/certs/generated/client_from_root.key.pem +50 -50
- data/spec/fixtures/certs/generated/client_from_root.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/client_from_root.p12 +0 -0
- data/spec/fixtures/certs/generated/client_from_root.pem +28 -28
- data/spec/fixtures/certs/generated/client_from_untrusted.jks +0 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.key.pem +50 -50
- data/spec/fixtures/certs/generated/client_from_untrusted.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/client_from_untrusted.p12 +0 -0
- data/spec/fixtures/certs/generated/client_from_untrusted.pem +28 -28
- data/spec/fixtures/certs/generated/client_self_signed.jks +0 -0
- data/spec/fixtures/certs/generated/client_self_signed.key.pem +50 -50
- data/spec/fixtures/certs/generated/client_self_signed.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/client_self_signed.p12 +0 -0
- data/spec/fixtures/certs/generated/client_self_signed.pem +28 -28
- data/spec/fixtures/certs/generated/root.key.pem +50 -50
- data/spec/fixtures/certs/generated/root.pem +28 -28
- data/spec/fixtures/certs/generated/server_from_root-key-pkcs8.pem +50 -50
- data/spec/fixtures/certs/generated/server_from_root.jks +0 -0
- data/spec/fixtures/certs/generated/server_from_root.key.pem +50 -50
- data/spec/fixtures/certs/generated/server_from_root.key.pkcs8.pem +52 -52
- data/spec/fixtures/certs/generated/server_from_root.p12 +0 -0
- data/spec/fixtures/certs/generated/server_from_root.pem +29 -29
- data/spec/fixtures/certs/generated/untrusted.key.pem +50 -50
- data/spec/fixtures/certs/generated/untrusted.pem +28 -28
- data/spec/unit/full_transmission_spec.rb +10 -2
- data/spec/unit/load_balancer_spec.rb +67 -0
- data/spec/unit/logstash_output_spec.rb +178 -17
- metadata +22 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 46207814a176a2b931e8ce99b90c00f6b5dd7702369743e639d8e98a6aa8d207
|
4
|
+
data.tar.gz: c551712f4de10c93a1220df7d0aac3d4495e49e7a75a8b20f4fb5e57f10e408c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 20a1f184bd2b45be391f0d8f2e32eb4af5386881ab40eead41328eef107abcc204b36b06fe859006371ae56a80f54217f4c4954b16decd40040e6f53ccf26649
|
7
|
+
data.tar.gz: 692844a4744aab3bd8de0688f0a53cd3cf7adf9234ad0df7c7fe5c6db8b23c7b9eeb3ff82c5a5f5eb09d978442fe9346c03055eaf86c0c9f02d182324474d33d
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 1.0.0
|
2
|
+
- Introduces the load balancing mechanism to distribute the requests among the `hosts` [#16](https://github.com/logstash-plugins/logstash-integration-logstash/pull/16)
|
3
|
+
|
1
4
|
## 0.0.5
|
2
5
|
- [DOC] Fixes to link formatting [#15](https://github.com/logstash-plugins/logstash-integration-logstash/pull/15)
|
3
6
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0
|
1
|
+
1.0.0
|
data/docs/index.asciidoc
CHANGED
@@ -21,7 +21,7 @@ include::{include_path}/plugin_header.asciidoc[]
|
|
21
21
|
|
22
22
|
==== Description
|
23
23
|
|
24
|
-
The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another:
|
24
|
+
The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another instance(s):
|
25
25
|
|
26
26
|
* {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin]
|
27
27
|
* {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin]
|
@@ -29,7 +29,7 @@ The Logstash Integration Plugin provides integrated plugins for sending events f
|
|
29
29
|
[id="plugins-{type}s-{plugin}-concepts"]
|
30
30
|
===== High-level concepts
|
31
31
|
|
32
|
-
You can configure a `logstash` output to send events to
|
32
|
+
You can configure a `logstash` output to send events to one or more `logstash` inputs, which are each in another pipeline that is running in different processes or on a different host.
|
33
33
|
|
34
34
|
To do so, you should first configure the downstream pipeline with a `logstash` input plugin, bound to an available port so that it can listen for inbound connections.
|
35
35
|
Security is enabled by default, so you will need to either provide identity material or disable SSL.
|
@@ -53,13 +53,11 @@ input {
|
|
53
53
|
Once the downstream pipeline is configured and running, you may send events from any number of upstream pipelines by adding a `logstash` output plugin that points to the downstream input.
|
54
54
|
You may need to configure SSL to trust the certificates presented by the downstream input plugin.
|
55
55
|
|
56
|
-
NOTE: Single host endpoint is supported for `hosts`. Multi-host support is coming soon.
|
57
|
-
|
58
56
|
[source]
|
59
57
|
----
|
60
58
|
output {
|
61
59
|
logstash {
|
62
|
-
hosts => "10.0.0.123:9800"
|
60
|
+
hosts => ["10.0.0.123:9800", "10.0.0.125:9801"]
|
63
61
|
|
64
62
|
# SSL TRUST <1>
|
65
63
|
ssl_truststore_path => "/path/to/truststore.p12"
|
@@ -69,4 +67,9 @@ output {
|
|
69
67
|
----
|
70
68
|
<1> Unless SSL is disabled or the downstream input is expected to present certificates signed by globally-trusted authorities, you will likely need to provide a source-of-trust.
|
71
69
|
|
70
|
+
[id="plugins-{type}s-{plugin}-load-balancing"]
|
71
|
+
==== Load Balancing
|
72
|
+
|
73
|
+
When a `logstash` output is configured to send to multiple `hosts`, it distributes events in batches to _all_ of those downstream hosts fairly, favoring those without recent errors. This increases the likelihood of each batch being routed to a downstream that is up and has capacity to receive events.
|
74
|
+
|
72
75
|
:no_codec!:
|
@@ -103,7 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
103
103
|
[cols="<,<,<",options="header",]
|
104
104
|
|=======================================================================
|
105
105
|
|Setting |Input type |Required
|
106
|
-
| <<plugins-{type}s-{plugin}-hosts>>
|
106
|
+
| <<plugins-{type}s-{plugin}-hosts>> |list of <<string,string>> |Yes
|
107
107
|
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
108
108
|
| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
|
109
109
|
| <<plugins-{type}s-{plugin}-ssl_certificate>> | <<path,path>>|No
|
@@ -125,27 +125,27 @@ output plugins.
|
|
125
125
|
[id="plugins-{type}s-{plugin}-hosts"]
|
126
126
|
===== `hosts`
|
127
127
|
|
128
|
-
* Value type is <<string,string>>
|
128
|
+
* Value type is list of <<string,string>>
|
129
129
|
* There is no default value for this setting.
|
130
130
|
* Constraints:
|
131
131
|
** When using IPv6, IP address must be in an enclosed in brackets.
|
132
132
|
** When a port is not provided, the default `9800` is used.
|
133
133
|
|
134
|
-
|
135
|
-
|
136
|
-
NOTE: Single host endpoint is supported for `hosts`. Multi-host support is coming soon.
|
134
|
+
The addresses of one or more downstream `input`s to connect to.
|
137
135
|
|
138
136
|
Host can be any of IPv4, IPv6 (in enclosed bracket) or host name, examples:
|
139
137
|
|
140
138
|
* `"127.0.0.1"`
|
141
139
|
* `"127.0.0.1:9801"`
|
142
140
|
* `"ds.example.com"`
|
143
|
-
* `"ds.example:9802"`
|
141
|
+
* `"ds.example.com:9802"`
|
144
142
|
* `"[::1]"`
|
145
143
|
* `"[::1]:9803"`
|
146
144
|
* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"`
|
147
145
|
* `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9804"`
|
148
146
|
|
147
|
+
Plugin balances incoming load among the `hosts`. For more information, visit {logstash-ref}/plugins-integrations-logstash.html[Logstash integration plugin] _Load Balancing_ section.
|
148
|
+
|
149
149
|
When connecting, communication to downstream input {ls} is secured with SSL unless configured otherwise.
|
150
150
|
|
151
151
|
[WARNING]
|
@@ -1,58 +1,59 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'logstash/outputs/base'
|
4
|
-
require 'logstash/namespace'
|
5
3
|
|
6
|
-
require "logstash/
|
4
|
+
require "logstash/outputs/base"
|
5
|
+
require "logstash/namespace"
|
6
|
+
|
7
|
+
require 'logstash/plugin_mixins/normalize_config_support'
|
8
|
+
require "logstash/plugin_mixins/http_client"
|
7
9
|
require "logstash/plugin_mixins/validator_support/required_host_optional_port_validation_adapter"
|
10
|
+
require "zlib"
|
11
|
+
|
12
|
+
require "stud/interval" # Stud::stoppable_sleep
|
8
13
|
|
9
14
|
class LogStash::Outputs::Logstash < LogStash::Outputs::Base
|
10
15
|
extend LogStash::PluginMixins::ValidatorSupport::RequiredHostOptionalPortValidationAdapter
|
11
16
|
|
12
|
-
include LogStash::PluginMixins::
|
17
|
+
include LogStash::PluginMixins::HttpClient[:with_deprecated => false]
|
18
|
+
include LogStash::PluginMixins::NormalizeConfigSupport
|
19
|
+
|
20
|
+
require "logstash/utils/load_balancer"
|
13
21
|
|
14
22
|
config_name "logstash"
|
15
23
|
|
16
24
|
# Sets the host of the downstream Logstash instance.
|
17
25
|
# Host can be any of IPv4, IPv6 (requires to be in enclosed bracket) or host name, the forms:
|
18
|
-
# `"127.0.0.1"`
|
19
|
-
# `"127.0.0.1:
|
20
|
-
# `"foo-bar.com"`
|
21
|
-
# `"
|
22
|
-
# `"[::1]"`
|
23
|
-
# `"[::1]:9000"`
|
26
|
+
# `"127.0.0.1"` or `["127.0.0.1"]` if single host with default port
|
27
|
+
# `"127.0.0.1:9801"` or `["127.0.0.1:9801"]` if single host with custom port
|
28
|
+
# `["foo-bar.com", "foo-bar.com:9800"]`
|
29
|
+
# `["[::1]", "[::1]:9000"]`
|
24
30
|
# `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"`
|
25
31
|
#
|
26
|
-
|
27
|
-
#
|
28
|
-
config :hosts, :validate => :required_host_optional_port, :required => true
|
29
|
-
|
30
|
-
# optional username/password credentials
|
31
|
-
config :username, :validate => :string, :required => false
|
32
|
-
config :password, :validate => :password, :required => false
|
32
|
+
config :hosts, :validate => :required_host_optional_port, :list => true, :required => true
|
33
33
|
|
34
|
-
config :
|
34
|
+
config :username, :validate => :string, :required => false
|
35
35
|
|
36
|
-
|
37
|
-
config :ssl_certificate, :validate => :path
|
38
|
-
config :ssl_key, :validate => :path
|
36
|
+
config :ssl_enabled, :validate => :boolean, :default => true
|
39
37
|
|
40
|
-
|
41
|
-
config :ssl_keystore_path, :validate => :path
|
42
|
-
config :ssl_keystore_password, :validate => :password
|
38
|
+
config :user, :validate => :string, :deprecated => "Use `username` instead.", :required => false
|
43
39
|
|
44
|
-
|
45
|
-
config :ssl_verification_mode, :validate => %w(full none), :default => 'full'
|
46
|
-
|
47
|
-
# SSL:TRUST:SOURCE ca file
|
48
|
-
config :ssl_certificate_authorities, :validate => :path, :list => true
|
49
|
-
|
50
|
-
# SSL:TRUST:SOURCE truststore
|
51
|
-
config :ssl_truststore_path, :validate => :path
|
52
|
-
config :ssl_truststore_password, :validate => :password
|
40
|
+
DEFAULT_PORT = 9800.freeze
|
53
41
|
|
54
|
-
|
55
|
-
|
42
|
+
RETRIABLE_CODES = [429, 500, 502, 503, 504]
|
43
|
+
RETRYABLE_MANTICORE_EXCEPTIONS = [
|
44
|
+
::Manticore::Timeout,
|
45
|
+
::Manticore::SocketException,
|
46
|
+
::Manticore::ClientProtocolException,
|
47
|
+
::Manticore::ResolutionFailure,
|
48
|
+
::Manticore::SocketTimeout
|
49
|
+
]
|
50
|
+
RETRYABLE_EXCEPTION_PATTERN = Regexp.union([
|
51
|
+
/Connection reset by peer/i,
|
52
|
+
/Read Timed out/i,
|
53
|
+
])
|
54
|
+
|
55
|
+
# @api private
|
56
|
+
attr_reader :http_client
|
56
57
|
|
57
58
|
def initialize(*a)
|
58
59
|
super
|
@@ -61,129 +62,252 @@ class LogStash::Outputs::Logstash < LogStash::Outputs::Base
|
|
61
62
|
fail LogStash::ConfigurationError, 'The `logstash` output does not have an externally-configurable `codec`'
|
62
63
|
end
|
63
64
|
|
64
|
-
|
65
|
-
|
66
|
-
|
65
|
+
@headers = {
|
66
|
+
"Content-Type" => "application/x-ndjson".freeze,
|
67
|
+
"Content-Encoding" => "gzip".freeze
|
68
|
+
}.freeze
|
67
69
|
|
68
|
-
logger.debug("
|
69
|
-
@internal_http = plugin_factory.output('http').new(inner_http_output_options)
|
70
|
-
logger.debug("inner HTTP output plugin has been initialized")
|
70
|
+
logger.debug("`logstash` output plugin has been initialized")
|
71
71
|
end
|
72
72
|
|
73
73
|
def register
|
74
|
-
logger.debug("
|
75
|
-
|
76
|
-
|
74
|
+
logger.debug("Registering `logstash` output plugin")
|
75
|
+
|
76
|
+
@username = normalize_config(:username) do |normalize|
|
77
|
+
normalize.with_deprecated_alias(:user)
|
78
|
+
end
|
79
|
+
# remove after deprecating user in the http-mixin
|
80
|
+
@user = @username ? @username.freeze : @user
|
81
|
+
|
82
|
+
validate_auth_settings!
|
83
|
+
|
84
|
+
if @ssl_enabled == false
|
85
|
+
rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
|
86
|
+
fail(LogStash::ConfigurationError, "Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
|
87
|
+
end
|
88
|
+
|
89
|
+
validate_ssl_identity_options!
|
90
|
+
validate_ssl_trust_options!
|
91
|
+
|
92
|
+
# if we don't initialize now, we get runtime error when sending events if there are issues with configs
|
93
|
+
@http_client = client
|
94
|
+
fail(LogStash::ConfigurationError, "`hosts` must not be empty") if @hosts.empty?
|
95
|
+
|
96
|
+
@load_balancer = LoadBalancer.new(normalize_host_uris)
|
97
|
+
|
98
|
+
logger.debug("`logstash` output plugin has been registered")
|
99
|
+
end
|
100
|
+
|
101
|
+
def validate_auth_settings!
|
102
|
+
if @username
|
103
|
+
fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided') if @password.nil?
|
104
|
+
logger.warn("Transmitting credentials over non-secured connection") if @ssl_enabled == false
|
105
|
+
elsif @password
|
106
|
+
fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def validate_ssl_identity_options!
|
111
|
+
if @ssl_certificate && @ssl_keystore_path
|
112
|
+
fail(LogStash::ConfigurationError, "SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both")
|
113
|
+
elsif @ssl_certificate
|
114
|
+
fail(LogStash::ConfigurationError, "`ssl_key` is REQUIRED when `ssl_certificate` is provided") if @ssl_key.nil?
|
115
|
+
elsif @ssl_key
|
116
|
+
fail(LogStash::ConfigurationError, "`ssl_key` is not allowed unless `ssl_certificate` is configured")
|
117
|
+
elsif @ssl_keystore_path
|
118
|
+
fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided") if @ssl_keystore_password.nil?
|
119
|
+
elsif @ssl_keystore_password
|
120
|
+
fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
|
121
|
+
else
|
122
|
+
# acceptable
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def validate_ssl_trust_options!
|
127
|
+
if @ssl_certificate_authorities&.any? && @ssl_truststore_path
|
128
|
+
fail(LogStash::ConfigurationError, "SSL trust can be configured with EITHER `ssl_certificate_authorities` OR `ssl_truststore_*`, but not both")
|
129
|
+
elsif @ssl_certificate_authorities&.any?
|
130
|
+
fail(LogStash::ConfigurationError, "SSL Certificate Authorities cannot be configured when `ssl_verification_mode => none`") if @ssl_verification_mode == 'none'
|
131
|
+
elsif @ssl_truststore_path
|
132
|
+
fail(LogStash::ConfigurationError, "SSL Truststore cannot be configured when `ssl_verification_mode => none`") if @ssl_verification_mode == 'none'
|
133
|
+
fail(LogStash::ConfigurationError, "`ssl_truststore_password` is REQUIRED when `ssl_truststore_path` is provided") if @ssl_truststore_password.nil?
|
134
|
+
elsif @ssl_truststore_password
|
135
|
+
fail(LogStash::ConfigurationError, "`ssl_truststore_password` not allowed unless `ssl_truststore_path` is configured")
|
136
|
+
end
|
77
137
|
end
|
78
138
|
|
79
139
|
def multi_receive(events)
|
80
140
|
return if events.empty?
|
81
|
-
|
82
|
-
|
83
|
-
rescue => e
|
84
|
-
logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
|
85
|
-
raise
|
141
|
+
|
142
|
+
send_events(events)
|
86
143
|
end
|
87
144
|
|
88
145
|
def stop
|
89
|
-
logger.debug("
|
90
|
-
@internal_http.stop
|
91
|
-
logger.debug('inner HTTP output plugin has been stopped')
|
146
|
+
logger.debug("`logstash` output plugin has been stopped")
|
92
147
|
end
|
93
148
|
|
94
149
|
def close
|
95
|
-
logger.debug("
|
96
|
-
|
97
|
-
logger.debug(
|
150
|
+
logger.debug("Closing `logstash` output plugin")
|
151
|
+
http_client.close
|
152
|
+
logger.debug("`logstash` output plugin has been closed")
|
98
153
|
end
|
99
154
|
|
100
|
-
|
155
|
+
private
|
101
156
|
|
102
|
-
def
|
103
|
-
@
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
'http_compression' => true,
|
114
|
-
}
|
115
|
-
|
116
|
-
if @username
|
117
|
-
http_options['user'] = @username
|
118
|
-
http_options['password'] = @password || fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided')
|
119
|
-
logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
|
120
|
-
elsif @password
|
121
|
-
fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
|
122
|
-
end
|
157
|
+
def normalize_host_uris
|
158
|
+
@_normalized_host_uris ||= begin
|
159
|
+
scheme = @ssl_enabled ? 'https' : 'http'
|
160
|
+
@hosts.map do |destination| # Struct(:host,:port)
|
161
|
+
URI::Generic.build(:scheme => scheme,
|
162
|
+
:host => destination.host,
|
163
|
+
:port => destination.port || DEFAULT_PORT)
|
164
|
+
end.map(&:to_s).map(&:freeze)
|
165
|
+
end
|
166
|
+
end
|
123
167
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
else
|
128
|
-
http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
|
168
|
+
def send_events(events)
|
169
|
+
body = LogStash::Json.dump(events.map(&:to_hash))
|
170
|
+
compressed_body = gzip(body)
|
129
171
|
|
130
|
-
|
131
|
-
|
132
|
-
|
172
|
+
next_backoff = 0.1
|
173
|
+
max_backoff = 30
|
174
|
+
|
175
|
+
loop do
|
176
|
+
next_action = transmit(body, compressed_body)
|
177
|
+
break unless next_action == :retry
|
133
178
|
|
134
|
-
|
179
|
+
Stud.stoppable_sleep(next_backoff) { pipeline_shutdown_requested? }
|
180
|
+
next_backoff = [next_backoff*2, max_backoff].min
|
181
|
+
|
182
|
+
if pipeline_shutdown_requested?
|
183
|
+
logger.warn "Aborting the batch due to shutdown request"
|
184
|
+
abort_batch_if_available!
|
185
|
+
break # legacy abort (lossy)
|
186
|
+
end
|
135
187
|
end
|
188
|
+
rescue => e
|
189
|
+
# This should never happen unless there's a flat out bug in the code
|
190
|
+
logger.error("Error occurred while sending events",
|
191
|
+
:class => e.class.name,
|
192
|
+
:message => e.message,
|
193
|
+
:backtrace => e.backtrace)
|
194
|
+
raise e
|
136
195
|
end
|
137
196
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
197
|
+
# The exceptions we decide (retriable or abort) to let the Load Balancer know
|
198
|
+
TransmitException = Class.new(RuntimeError)
|
199
|
+
private_constant :TransmitException
|
200
|
+
|
201
|
+
RetriableTransmitException = Class.new(TransmitException)
|
202
|
+
private_constant :RetriableTransmitException
|
203
|
+
|
204
|
+
TerminalTransmitException = Class.new(TransmitException)
|
205
|
+
private_constant :TerminalTransmitException
|
206
|
+
|
207
|
+
##
|
208
|
+
# @param body [String]
|
209
|
+
# @param compressed_body [String]
|
210
|
+
# @return [:done, :abort, :retry]
|
211
|
+
def transmit(body, compressed_body)
|
212
|
+
@load_balancer.select do |selected_host_uri|
|
213
|
+
response = begin
|
214
|
+
http_client.post(selected_host_uri, :body => compressed_body, :headers => @headers).call
|
215
|
+
rescue => exception
|
216
|
+
retryable_exception = retryable_exception?(exception)
|
217
|
+
log_exception(selected_host_uri, exception, body, retryable_exception)
|
218
|
+
|
219
|
+
# raise exception to mar the host error
|
220
|
+
raise retryable_exception ? RetriableTransmitException : TerminalTransmitException
|
221
|
+
end
|
222
|
+
|
223
|
+
return :done if response_success?(response.code)
|
224
|
+
|
225
|
+
retryable_response = retryable_response?(response.code)
|
226
|
+
log_response(selected_host_uri, response, body, retryable_response)
|
227
|
+
|
228
|
+
# raise exception to mar the host error
|
229
|
+
raise retryable_response ? RetriableTransmitException : TerminalTransmitException
|
230
|
+
end
|
231
|
+
rescue RetriableTransmitException => exception
|
232
|
+
return :retry
|
233
|
+
rescue TerminalTransmitException => exception
|
234
|
+
return :abort
|
145
235
|
end
|
146
236
|
|
147
|
-
def
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
elsif @ssl_key
|
156
|
-
fail(LogStash::ConfigurationError, '`ssl_key` is not allowed unless `ssl_certificate` is configured')
|
157
|
-
elsif @ssl_keystore_path
|
158
|
-
return {
|
159
|
-
'ssl_keystore_path' => @ssl_keystore_path,
|
160
|
-
'ssl_keystore_password' => @ssl_keystore_password || fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided"),
|
161
|
-
}
|
162
|
-
elsif @ssl_keystore_password
|
163
|
-
fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
|
237
|
+
def log_response(uri, response, body, retriable)
|
238
|
+
response_code = response.code
|
239
|
+
if retriable
|
240
|
+
if response_code == 429
|
241
|
+
logger.debug("Encountered a retriable 429 response")
|
242
|
+
else
|
243
|
+
logger.warn("Encountered a retryable error in `logstash` output", :code => response_code, :body => response.body)
|
244
|
+
end
|
164
245
|
else
|
165
|
-
|
246
|
+
logger.error("Encountered error",
|
247
|
+
:response_code => response_code,
|
248
|
+
:host => uri,
|
249
|
+
:body => body
|
250
|
+
)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
def log_exception(uri, exception, body, retriable)
|
255
|
+
log_entry = { :host => uri, :message => exception.message, :class => exception.class, :retry => retriable }
|
256
|
+
if logger.debug?
|
257
|
+
# backtraces are big
|
258
|
+
log_entry[:backtrace] = exception.backtrace
|
259
|
+
# body can be big and may have sensitive data
|
260
|
+
log_entry[:body] = body
|
166
261
|
end
|
262
|
+
logger.error("Could not send data to host", log_entry)
|
167
263
|
end
|
168
264
|
|
169
|
-
def
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
265
|
+
def gzip(data)
|
266
|
+
gz = StringIO.new
|
267
|
+
gz.set_encoding("BINARY")
|
268
|
+
z = Zlib::GzipWriter.new(gz)
|
269
|
+
z.write(data)
|
270
|
+
z.close
|
271
|
+
gz.string
|
272
|
+
end
|
177
273
|
|
178
|
-
|
179
|
-
|
180
|
-
|
274
|
+
def response_success?(response_code)
|
275
|
+
response_code >= 200 && response_code <= 299
|
276
|
+
end
|
181
277
|
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
278
|
+
def retryable_exception?(exception)
|
279
|
+
retryable_manticore_exception?(exception) || retryable_unknown_exception?(exception)
|
280
|
+
end
|
281
|
+
|
282
|
+
def retryable_manticore_exception?(exception)
|
283
|
+
RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me)}
|
284
|
+
end
|
285
|
+
|
286
|
+
def retryable_unknown_exception?(exception)
|
287
|
+
exception.is_a?(::Manticore::UnknownException) &&
|
288
|
+
RETRYABLE_EXCEPTION_PATTERN.match?(exception.message)
|
289
|
+
end
|
290
|
+
|
291
|
+
def retryable_response?(response_code)
|
292
|
+
RETRIABLE_CODES.include?(response_code)
|
293
|
+
end
|
294
|
+
|
295
|
+
# Emulate `pipeline_shutdown_requested?` when running on older Logstash
|
296
|
+
unless ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.1.0')
|
297
|
+
def pipeline_shutdown_requested?
|
298
|
+
execution_context&.pipeline&.shutdown_requested?
|
299
|
+
end
|
300
|
+
end
|
301
|
+
|
302
|
+
# When running on Logstash that can abort batches,
|
303
|
+
# raise the required exception, do nothing otherwise.
|
304
|
+
if ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.8.0')
|
305
|
+
def abort_batch_if_available!
|
306
|
+
raise org.logstash.execution.AbortedBatchException.new
|
307
|
+
end
|
308
|
+
else
|
309
|
+
def abort_batch_if_available!
|
310
|
+
nil
|
187
311
|
end
|
188
312
|
end
|
189
313
|
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
|
4
|
+
require "monitor"
|
5
|
+
|
6
|
+
class LoadBalancer
|
7
|
+
include MonitorMixin
|
8
|
+
|
9
|
+
##
|
10
|
+
# Creates a new Router with the provided downstream_infos
|
11
|
+
# that ignores errors older than the cool_off period
|
12
|
+
# @param host_infos [Enumerable<HostSate>]: a list of downstream hosts
|
13
|
+
# to include in routing
|
14
|
+
# @param cool_off [Integer]: The cool_off period in seconds in which downstreams with
|
15
|
+
# recent errors are de-prioritized (default: 60)
|
16
|
+
def initialize(host_infos, cool_off: 60)
|
17
|
+
super() # to initialize MonitorMixin
|
18
|
+
|
19
|
+
fail ArgumentError, "Non-empty `host_infos` hosts required." unless host_infos&.any?
|
20
|
+
fail ArgumentError, "`cool_off` requires integer value." unless cool_off.kind_of?(Integer)
|
21
|
+
|
22
|
+
@cool_off = cool_off
|
23
|
+
@host_states = host_infos.map do |host_info|
|
24
|
+
HostState.new(host_info)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
##
|
29
|
+
# Yields the block with a {HostState}, prioritizing
|
30
|
+
# hosts that are less concurrently-used and which have
|
31
|
+
# not errored recently.
|
32
|
+
# @yield param selected [HostState]
|
33
|
+
def select
|
34
|
+
selected = synchronize { pick_one.tap(&:increment) }
|
35
|
+
yield selected.uri
|
36
|
+
rescue
|
37
|
+
synchronize { selected.mark_error }
|
38
|
+
raise
|
39
|
+
ensure
|
40
|
+
synchronize { selected.decrement }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def pick_one
|
46
|
+
threshold = Time.now.to_i - @cool_off
|
47
|
+
@host_states.sort_by do |host_state|
|
48
|
+
[
|
49
|
+
[host_state.last_error, threshold].max, # deprioritize recent errors
|
50
|
+
host_state.concurrent, # deprioritize high concurrency
|
51
|
+
host_state.last_start # deprioritize recent use
|
52
|
+
]
|
53
|
+
end.first
|
54
|
+
end
|
55
|
+
|
56
|
+
class HostState
|
57
|
+
def initialize(host_uri)
|
58
|
+
@uri = host_uri
|
59
|
+
@last_error = 0
|
60
|
+
@concurrent = 0
|
61
|
+
@last_start = 0
|
62
|
+
end
|
63
|
+
attr_reader :uri
|
64
|
+
attr_reader :last_error
|
65
|
+
attr_reader :concurrent
|
66
|
+
attr_reader :last_start
|
67
|
+
|
68
|
+
def increment
|
69
|
+
@concurrent += 1
|
70
|
+
@last_start = Time.now.to_f
|
71
|
+
end
|
72
|
+
|
73
|
+
def decrement
|
74
|
+
@concurrent -= 1
|
75
|
+
end
|
76
|
+
|
77
|
+
def mark_error
|
78
|
+
@last_error = Time.now.to_i
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -28,8 +28,9 @@ Gem::Specification.new do |s|
|
|
28
28
|
s.add_runtime_dependency "logstash-mixin-validator_support", "~> 1.1"
|
29
29
|
s.add_runtime_dependency "logstash-codec-json_lines", "~> 3.1"
|
30
30
|
|
31
|
+
s.add_runtime_dependency "logstash-mixin-http_client", "~> 7.3"
|
31
32
|
s.add_runtime_dependency "logstash-input-http", ">= 3.7.0" # some params not available in older versions because they are renamed, such as `cacert` to `ssl_certificate_authorities`
|
32
|
-
s.add_runtime_dependency "
|
33
|
+
s.add_runtime_dependency "stud"
|
33
34
|
|
34
35
|
s.add_development_dependency "logstash-devutils"
|
35
36
|
s.add_development_dependency "rspec-collection_matchers"
|
Binary file
|