logstash-output-datadog_logs 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/datadog_logs.rb +42 -15
- data/logstash-output-datadog_logs.gemspec +2 -1
- data/spec/outputs/datadog_logs_spec.rb +97 -1
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8148c91e062cc8e1999b780d95f529d9cd1516664f7872a96da51e1656c2b71d
|
4
|
+
data.tar.gz: 6fd0ddbf65b8240e1609c0c74a57f7ffcf50e7945cf2601484007725ff5465b2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ce5a190b7c1550eb4edcec848aee451abd9814894da3e9c4302611c83c1e9414373086b69e60372bfde6cd6ad3da611146ac82413563b95c49721bdb0a024c68
|
7
|
+
data.tar.gz: 33c1d8afda9da935ccb5f9ee43388f424ab7c8fefdb61b16ef9ac474f5d1c5834c053f911bfadce596291ea62148ff0f0a1beaf7bf56819ad433ac7fea92ae71
|
data/CHANGELOG.md
CHANGED
@@ -50,15 +50,19 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
50
50
|
def multi_receive(events)
|
51
51
|
return if events.empty?
|
52
52
|
encoded_events = @codec.multi_encode(events)
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
53
|
+
begin
|
54
|
+
if @use_http
|
55
|
+
batches = batch_http_events(encoded_events, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
|
56
|
+
batches.each do |batched_event|
|
57
|
+
process_encoded_payload(format_http_event_batch(batched_event))
|
58
|
+
end
|
59
|
+
else
|
60
|
+
encoded_events.each do |encoded_event|
|
61
|
+
process_encoded_payload(format_tcp_event(encoded_event.last, @api_key, DD_MAX_BATCH_SIZE))
|
62
|
+
end
|
61
63
|
end
|
64
|
+
rescue => e
|
65
|
+
@logger.error("Uncaught processing exception in datadog forwarder #{e.message}")
|
62
66
|
end
|
63
67
|
end
|
64
68
|
|
@@ -158,12 +162,14 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
158
162
|
send(payload)
|
159
163
|
rescue RetryableError => e
|
160
164
|
if retries < max_retries || max_retries < 0
|
161
|
-
@logger.warn("Retrying
|
165
|
+
@logger.warn("Retrying send due to: #{e.message}")
|
162
166
|
sleep backoff
|
163
167
|
backoff = 2 * backoff unless backoff > max_backoff
|
164
168
|
retries += 1
|
165
169
|
retry
|
166
170
|
end
|
171
|
+
rescue => ex
|
172
|
+
@logger.error("Unmanaged exception while sending log to datadog #{ex.message}")
|
167
173
|
end
|
168
174
|
end
|
169
175
|
|
@@ -179,6 +185,13 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
179
185
|
class DatadogHTTPClient < DatadogClient
|
180
186
|
require "manticore"
|
181
187
|
|
188
|
+
RETRYABLE_EXCEPTIONS = [
|
189
|
+
::Manticore::Timeout,
|
190
|
+
::Manticore::SocketException,
|
191
|
+
::Manticore::ClientProtocolException,
|
192
|
+
::Manticore::ResolutionFailure
|
193
|
+
]
|
194
|
+
|
182
195
|
def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
|
183
196
|
@logger = logger
|
184
197
|
protocol = use_ssl ? "https" : "http"
|
@@ -194,13 +207,27 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
194
207
|
end
|
195
208
|
|
196
209
|
def send(payload)
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
210
|
+
begin
|
211
|
+
response = @client.post(@url, :body => payload, :headers => @headers).call
|
212
|
+
if response.code >= 500
|
213
|
+
raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
|
214
|
+
end
|
215
|
+
if response.code >= 400
|
216
|
+
@logger.error("Unable to send payload due to client error: #{response.code} #{response.body}")
|
217
|
+
end
|
218
|
+
rescue => client_exception
|
219
|
+
should_retry = retryable_exception?(client_exception)
|
220
|
+
if should_retry
|
221
|
+
raise RetryableError.new "Unable to send payload #{client_exception.message}"
|
222
|
+
else
|
223
|
+
raise client_exception
|
224
|
+
end
|
203
225
|
end
|
226
|
+
|
227
|
+
end
|
228
|
+
|
229
|
+
def retryable_exception?(exception)
|
230
|
+
RETRYABLE_EXCEPTIONS.any? { |e| exception.is_a?(e) }
|
204
231
|
end
|
205
232
|
|
206
233
|
def close
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-datadog_logs'
|
3
|
-
s.version = '0.4.
|
3
|
+
s.version = '0.4.1'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
|
6
6
|
s.homepage = 'https://www.datadoghq.com/'
|
@@ -22,4 +22,5 @@ Gem::Specification.new do |s|
|
|
22
22
|
s.add_runtime_dependency 'logstash-codec-json'
|
23
23
|
|
24
24
|
s.add_development_dependency 'logstash-devutils'
|
25
|
+
s.add_development_dependency 'webmock'
|
25
26
|
end
|
@@ -5,6 +5,7 @@
|
|
5
5
|
|
6
6
|
require "logstash/devutils/rspec/spec_helper"
|
7
7
|
require "logstash/outputs/datadog_logs"
|
8
|
+
require 'webmock/rspec'
|
8
9
|
|
9
10
|
describe LogStash::Outputs::DatadogLogs do
|
10
11
|
context "should register" do
|
@@ -79,7 +80,7 @@ describe LogStash::Outputs::DatadogLogs do
|
|
79
80
|
end
|
80
81
|
|
81
82
|
it "should truncate events whose length is bigger than the max request size" do
|
82
|
-
input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}),"foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
|
83
|
+
input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}), "foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
|
83
84
|
actual_events = subject.batch_http_events(input_events, 10, 3)
|
84
85
|
expect(actual_events.length).to eq(3)
|
85
86
|
expect(actual_events[0][0]).to eq("dd1")
|
@@ -88,6 +89,80 @@ describe LogStash::Outputs::DatadogLogs do
|
|
88
89
|
end
|
89
90
|
end
|
90
91
|
|
92
|
+
context "when facing HTTP connection issues" do
|
93
|
+
it "should retry when server is returning 5XX" do
|
94
|
+
api_key = 'XXX'
|
95
|
+
stub_dd_request_with_return_code(api_key, 500)
|
96
|
+
payload = '{}'
|
97
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
98
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should not retry when server is returning 4XX" do
|
102
|
+
api_key = 'XXX'
|
103
|
+
stub_dd_request_with_return_code(api_key, 400)
|
104
|
+
payload = '{}'
|
105
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
106
|
+
expect { client.send(payload) }.to_not raise_error
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should retry when facing a timeout exception from manticore" do
|
110
|
+
api_key = 'XXX'
|
111
|
+
stub_dd_request_with_error(api_key, Manticore::Timeout)
|
112
|
+
payload = '{}'
|
113
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
114
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should retry when facing a socket exception from manticore" do
|
118
|
+
api_key = 'XXX'
|
119
|
+
stub_dd_request_with_error(api_key, Manticore::SocketException)
|
120
|
+
payload = '{}'
|
121
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
122
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
123
|
+
end
|
124
|
+
|
125
|
+
it "should retry when facing a client protocol exception from manticore" do
|
126
|
+
api_key = 'XXX'
|
127
|
+
stub_dd_request_with_error(api_key, Manticore::ClientProtocolException)
|
128
|
+
payload = '{}'
|
129
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
130
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
131
|
+
end
|
132
|
+
|
133
|
+
it "should retry when facing a dns failure from manticore" do
|
134
|
+
api_key = 'XXX'
|
135
|
+
stub_dd_request_with_error(api_key, Manticore::ResolutionFailure)
|
136
|
+
payload = '{}'
|
137
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
138
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
139
|
+
end
|
140
|
+
|
141
|
+
it "should retry when facing a socket timeout from manticore" do
|
142
|
+
api_key = 'XXX'
|
143
|
+
stub_dd_request_with_error(api_key, Manticore::SocketTimeout)
|
144
|
+
payload = '{}'
|
145
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
146
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
147
|
+
end
|
148
|
+
|
149
|
+
it "should not retry when facing any other general error" do
|
150
|
+
api_key = 'XXX'
|
151
|
+
stub_dd_request_with_error(api_key, StandardError)
|
152
|
+
payload = '{}'
|
153
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
154
|
+
expect { client.send(payload) }.to raise_error(StandardError)
|
155
|
+
end
|
156
|
+
|
157
|
+
it "should not stop the forwarder when facing any client uncaught exception" do
|
158
|
+
api_key = 'XXX'
|
159
|
+
stub_dd_request_with_error(api_key, StandardError)
|
160
|
+
payload = '{}'
|
161
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
162
|
+
expect { client.send_retries(payload, 2, 2) }.to_not raise_error
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
91
166
|
context "when using TCP" do
|
92
167
|
it "should re-encode events" do
|
93
168
|
input_event = "{message=dd}"
|
@@ -101,4 +176,25 @@ describe LogStash::Outputs::DatadogLogs do
|
|
101
176
|
expect(encoded_event).to eq("xxx {...TRUNCATED...")
|
102
177
|
end
|
103
178
|
end
|
179
|
+
|
180
|
+
def stub_dd_request_with_return_code(api_key, return_code)
|
181
|
+
stub_dd_request(api_key).
|
182
|
+
to_return(status: return_code, body: "", headers: {})
|
183
|
+
end
|
184
|
+
|
185
|
+
def stub_dd_request_with_error(api_key, error)
|
186
|
+
stub_dd_request(api_key).
|
187
|
+
to_raise(error)
|
188
|
+
end
|
189
|
+
|
190
|
+
def stub_dd_request(api_key)
|
191
|
+
stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
|
192
|
+
with(
|
193
|
+
body: "{}",
|
194
|
+
headers: {
|
195
|
+
'Connection' => 'Keep-Alive',
|
196
|
+
'Content-Type' => 'application/json'
|
197
|
+
})
|
198
|
+
end
|
199
|
+
|
104
200
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-datadog_logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Datadog
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-03-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,6 +73,20 @@ dependencies:
|
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
name: webmock
|
83
|
+
prerelease: false
|
84
|
+
type: :development
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
76
90
|
description:
|
77
91
|
email: support@datadoghq.com
|
78
92
|
executables: []
|