logstash-output-datadog_logs 0.4.0 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6f81e4d1f9e76634f0ee83cc649ad18a9ee9defc51c54c0cd630e8c9bae69d53
4
- data.tar.gz: e15dfeba8935d842bc9db8ed8c1a8f166165376c4b0bc02d6feafca166313432
3
+ metadata.gz: 8148c91e062cc8e1999b780d95f529d9cd1516664f7872a96da51e1656c2b71d
4
+ data.tar.gz: 6fd0ddbf65b8240e1609c0c74a57f7ffcf50e7945cf2601484007725ff5465b2
5
5
  SHA512:
6
- metadata.gz: 46a621add6073375b653ce158b67d9aec0c1e6544c7d69ce7740b1b7930cb95a28525765048ebcc66975f9fa99b755f10fee32127a14b842ae0be80f034ebf11
7
- data.tar.gz: e9276dc886a503ed450a20ce986861835496b5f8f1d991389afd5ece5a0933b18ae2cad57d211a74d068b51024c560eabb7a859ba17ebd6b6f9ca44fe71fb2a2
6
+ metadata.gz: ce5a190b7c1550eb4edcec848aee451abd9814894da3e9c4302611c83c1e9414373086b69e60372bfde6cd6ad3da611146ac82413563b95c49721bdb0a024c68
7
+ data.tar.gz: 33c1d8afda9da935ccb5f9ee43388f424ab7c8fefdb61b16ef9ac474f5d1c5834c053f911bfadce596291ea62148ff0f0a1beaf7bf56819ad433ac7fea92ae71
@@ -1,3 +1,6 @@
1
+ ## 0.4.1
2
+ - Fix HTTP bug when remote server is timing out
3
+
1
4
  ## 0.4.0
2
5
  - Enable HTTP forwarding for logs
3
6
  - Provide an option to disable SSL hostname verification for HTTPS
@@ -50,15 +50,19 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
50
50
  def multi_receive(events)
51
51
  return if events.empty?
52
52
  encoded_events = @codec.multi_encode(events)
53
- if @use_http
54
- batches = batch_http_events(encoded_events, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
55
- batches.each do |batched_event|
56
- process_encoded_payload(format_http_event_batch(batched_event))
57
- end
58
- else
59
- encoded_events.each do |encoded_event|
60
- process_encoded_payload(format_tcp_event(encoded_event.last, @api_key, DD_MAX_BATCH_SIZE))
53
+ begin
54
+ if @use_http
55
+ batches = batch_http_events(encoded_events, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
56
+ batches.each do |batched_event|
57
+ process_encoded_payload(format_http_event_batch(batched_event))
58
+ end
59
+ else
60
+ encoded_events.each do |encoded_event|
61
+ process_encoded_payload(format_tcp_event(encoded_event.last, @api_key, DD_MAX_BATCH_SIZE))
62
+ end
61
63
  end
64
+ rescue => e
65
+ @logger.error("Uncaught processing exception in datadog forwarder #{e.message}")
62
66
  end
63
67
  end
64
68
 
@@ -158,12 +162,14 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
158
162
  send(payload)
159
163
  rescue RetryableError => e
160
164
  if retries < max_retries || max_retries < 0
161
- @logger.warn("Retrying ", :exception => e, :backtrace => e.backtrace)
165
+ @logger.warn("Retrying send due to: #{e.message}")
162
166
  sleep backoff
163
167
  backoff = 2 * backoff unless backoff > max_backoff
164
168
  retries += 1
165
169
  retry
166
170
  end
171
+ rescue => ex
172
+ @logger.error("Unmanaged exception while sending log to datadog #{ex.message}")
167
173
  end
168
174
  end
169
175
 
@@ -179,6 +185,13 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
179
185
  class DatadogHTTPClient < DatadogClient
180
186
  require "manticore"
181
187
 
188
+ RETRYABLE_EXCEPTIONS = [
189
+ ::Manticore::Timeout,
190
+ ::Manticore::SocketException,
191
+ ::Manticore::ClientProtocolException,
192
+ ::Manticore::ResolutionFailure
193
+ ]
194
+
182
195
  def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
183
196
  @logger = logger
184
197
  protocol = use_ssl ? "https" : "http"
@@ -194,13 +207,27 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
194
207
  end
195
208
 
196
209
  def send(payload)
197
- response = @client.post(@url, :body => payload, :headers => @headers).call
198
- if response.code >= 500
199
- raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
200
- end
201
- if response.code >= 400
202
- @logger.error("Unable to send payload due to client error: #{response.code} #{response.body}")
210
+ begin
211
+ response = @client.post(@url, :body => payload, :headers => @headers).call
212
+ if response.code >= 500
213
+ raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
214
+ end
215
+ if response.code >= 400
216
+ @logger.error("Unable to send payload due to client error: #{response.code} #{response.body}")
217
+ end
218
+ rescue => client_exception
219
+ should_retry = retryable_exception?(client_exception)
220
+ if should_retry
221
+ raise RetryableError.new "Unable to send payload #{client_exception.message}"
222
+ else
223
+ raise client_exception
224
+ end
203
225
  end
226
+
227
+ end
228
+
229
+ def retryable_exception?(exception)
230
+ RETRYABLE_EXCEPTIONS.any? { |e| exception.is_a?(e) }
204
231
  end
205
232
 
206
233
  def close
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-datadog_logs'
3
- s.version = '0.4.0'
3
+ s.version = '0.4.1'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
6
6
  s.homepage = 'https://www.datadoghq.com/'
@@ -22,4 +22,5 @@ Gem::Specification.new do |s|
22
22
  s.add_runtime_dependency 'logstash-codec-json'
23
23
 
24
24
  s.add_development_dependency 'logstash-devutils'
25
+ s.add_development_dependency 'webmock'
25
26
  end
@@ -5,6 +5,7 @@
5
5
 
6
6
  require "logstash/devutils/rspec/spec_helper"
7
7
  require "logstash/outputs/datadog_logs"
8
+ require 'webmock/rspec'
8
9
 
9
10
  describe LogStash::Outputs::DatadogLogs do
10
11
  context "should register" do
@@ -79,7 +80,7 @@ describe LogStash::Outputs::DatadogLogs do
79
80
  end
80
81
 
81
82
  it "should truncate events whose length is bigger than the max request size" do
82
- input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}),"foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
83
+ input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}), "foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
83
84
  actual_events = subject.batch_http_events(input_events, 10, 3)
84
85
  expect(actual_events.length).to eq(3)
85
86
  expect(actual_events[0][0]).to eq("dd1")
@@ -88,6 +89,80 @@ describe LogStash::Outputs::DatadogLogs do
88
89
  end
89
90
  end
90
91
 
92
+ context "when facing HTTP connection issues" do
93
+ it "should retry when server is returning 5XX" do
94
+ api_key = 'XXX'
95
+ stub_dd_request_with_return_code(api_key, 500)
96
+ payload = '{}'
97
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
98
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
99
+ end
100
+
101
+ it "should not retry when server is returning 4XX" do
102
+ api_key = 'XXX'
103
+ stub_dd_request_with_return_code(api_key, 400)
104
+ payload = '{}'
105
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
106
+ expect { client.send(payload) }.to_not raise_error
107
+ end
108
+
109
+ it "should retry when facing a timeout exception from manticore" do
110
+ api_key = 'XXX'
111
+ stub_dd_request_with_error(api_key, Manticore::Timeout)
112
+ payload = '{}'
113
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
114
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
115
+ end
116
+
117
+ it "should retry when facing a socket exception from manticore" do
118
+ api_key = 'XXX'
119
+ stub_dd_request_with_error(api_key, Manticore::SocketException)
120
+ payload = '{}'
121
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
122
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
123
+ end
124
+
125
+ it "should retry when facing a client protocol exception from manticore" do
126
+ api_key = 'XXX'
127
+ stub_dd_request_with_error(api_key, Manticore::ClientProtocolException)
128
+ payload = '{}'
129
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
130
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
131
+ end
132
+
133
+ it "should retry when facing a dns failure from manticore" do
134
+ api_key = 'XXX'
135
+ stub_dd_request_with_error(api_key, Manticore::ResolutionFailure)
136
+ payload = '{}'
137
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
138
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
139
+ end
140
+
141
+ it "should retry when facing a socket timeout from manticore" do
142
+ api_key = 'XXX'
143
+ stub_dd_request_with_error(api_key, Manticore::SocketTimeout)
144
+ payload = '{}'
145
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
146
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
147
+ end
148
+
149
+ it "should not retry when facing any other general error" do
150
+ api_key = 'XXX'
151
+ stub_dd_request_with_error(api_key, StandardError)
152
+ payload = '{}'
153
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
154
+ expect { client.send(payload) }.to raise_error(StandardError)
155
+ end
156
+
157
+ it "should not stop the forwarder when facing any client uncaught exception" do
158
+ api_key = 'XXX'
159
+ stub_dd_request_with_error(api_key, StandardError)
160
+ payload = '{}'
161
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
162
+ expect { client.send_retries(payload, 2, 2) }.to_not raise_error
163
+ end
164
+ end
165
+
91
166
  context "when using TCP" do
92
167
  it "should re-encode events" do
93
168
  input_event = "{message=dd}"
@@ -101,4 +176,25 @@ describe LogStash::Outputs::DatadogLogs do
101
176
  expect(encoded_event).to eq("xxx {...TRUNCATED...")
102
177
  end
103
178
  end
179
+
180
+ def stub_dd_request_with_return_code(api_key, return_code)
181
+ stub_dd_request(api_key).
182
+ to_return(status: return_code, body: "", headers: {})
183
+ end
184
+
185
+ def stub_dd_request_with_error(api_key, error)
186
+ stub_dd_request(api_key).
187
+ to_raise(error)
188
+ end
189
+
190
+ def stub_dd_request(api_key)
191
+ stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
192
+ with(
193
+ body: "{}",
194
+ headers: {
195
+ 'Connection' => 'Keep-Alive',
196
+ 'Content-Type' => 'application/json'
197
+ })
198
+ end
199
+
104
200
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-datadog_logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Datadog
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-02-25 00:00:00.000000000 Z
12
+ date: 2020-03-13 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -73,6 +73,20 @@ dependencies:
73
73
  - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
+ - !ruby/object:Gem::Dependency
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ name: webmock
83
+ prerelease: false
84
+ type: :development
85
+ version_requirements: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
76
90
  description:
77
91
  email: support@datadoghq.com
78
92
  executables: []