logstash-output-datadog_logs 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/datadog_logs.rb +42 -15
- data/logstash-output-datadog_logs.gemspec +2 -1
- data/spec/outputs/datadog_logs_spec.rb +97 -1
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8148c91e062cc8e1999b780d95f529d9cd1516664f7872a96da51e1656c2b71d
|
4
|
+
data.tar.gz: 6fd0ddbf65b8240e1609c0c74a57f7ffcf50e7945cf2601484007725ff5465b2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ce5a190b7c1550eb4edcec848aee451abd9814894da3e9c4302611c83c1e9414373086b69e60372bfde6cd6ad3da611146ac82413563b95c49721bdb0a024c68
|
7
|
+
data.tar.gz: 33c1d8afda9da935ccb5f9ee43388f424ab7c8fefdb61b16ef9ac474f5d1c5834c053f911bfadce596291ea62148ff0f0a1beaf7bf56819ad433ac7fea92ae71
|
data/CHANGELOG.md
CHANGED
@@ -50,15 +50,19 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
50
50
|
def multi_receive(events)
|
51
51
|
return if events.empty?
|
52
52
|
encoded_events = @codec.multi_encode(events)
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
53
|
+
begin
|
54
|
+
if @use_http
|
55
|
+
batches = batch_http_events(encoded_events, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
|
56
|
+
batches.each do |batched_event|
|
57
|
+
process_encoded_payload(format_http_event_batch(batched_event))
|
58
|
+
end
|
59
|
+
else
|
60
|
+
encoded_events.each do |encoded_event|
|
61
|
+
process_encoded_payload(format_tcp_event(encoded_event.last, @api_key, DD_MAX_BATCH_SIZE))
|
62
|
+
end
|
61
63
|
end
|
64
|
+
rescue => e
|
65
|
+
@logger.error("Uncaught processing exception in datadog forwarder #{e.message}")
|
62
66
|
end
|
63
67
|
end
|
64
68
|
|
@@ -158,12 +162,14 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
158
162
|
send(payload)
|
159
163
|
rescue RetryableError => e
|
160
164
|
if retries < max_retries || max_retries < 0
|
161
|
-
@logger.warn("Retrying
|
165
|
+
@logger.warn("Retrying send due to: #{e.message}")
|
162
166
|
sleep backoff
|
163
167
|
backoff = 2 * backoff unless backoff > max_backoff
|
164
168
|
retries += 1
|
165
169
|
retry
|
166
170
|
end
|
171
|
+
rescue => ex
|
172
|
+
@logger.error("Unmanaged exception while sending log to datadog #{ex.message}")
|
167
173
|
end
|
168
174
|
end
|
169
175
|
|
@@ -179,6 +185,13 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
179
185
|
class DatadogHTTPClient < DatadogClient
|
180
186
|
require "manticore"
|
181
187
|
|
188
|
+
RETRYABLE_EXCEPTIONS = [
|
189
|
+
::Manticore::Timeout,
|
190
|
+
::Manticore::SocketException,
|
191
|
+
::Manticore::ClientProtocolException,
|
192
|
+
::Manticore::ResolutionFailure
|
193
|
+
]
|
194
|
+
|
182
195
|
def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
|
183
196
|
@logger = logger
|
184
197
|
protocol = use_ssl ? "https" : "http"
|
@@ -194,13 +207,27 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
194
207
|
end
|
195
208
|
|
196
209
|
def send(payload)
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
210
|
+
begin
|
211
|
+
response = @client.post(@url, :body => payload, :headers => @headers).call
|
212
|
+
if response.code >= 500
|
213
|
+
raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
|
214
|
+
end
|
215
|
+
if response.code >= 400
|
216
|
+
@logger.error("Unable to send payload due to client error: #{response.code} #{response.body}")
|
217
|
+
end
|
218
|
+
rescue => client_exception
|
219
|
+
should_retry = retryable_exception?(client_exception)
|
220
|
+
if should_retry
|
221
|
+
raise RetryableError.new "Unable to send payload #{client_exception.message}"
|
222
|
+
else
|
223
|
+
raise client_exception
|
224
|
+
end
|
203
225
|
end
|
226
|
+
|
227
|
+
end
|
228
|
+
|
229
|
+
def retryable_exception?(exception)
|
230
|
+
RETRYABLE_EXCEPTIONS.any? { |e| exception.is_a?(e) }
|
204
231
|
end
|
205
232
|
|
206
233
|
def close
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-datadog_logs'
|
3
|
-
s.version = '0.4.
|
3
|
+
s.version = '0.4.1'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
|
6
6
|
s.homepage = 'https://www.datadoghq.com/'
|
@@ -22,4 +22,5 @@ Gem::Specification.new do |s|
|
|
22
22
|
s.add_runtime_dependency 'logstash-codec-json'
|
23
23
|
|
24
24
|
s.add_development_dependency 'logstash-devutils'
|
25
|
+
s.add_development_dependency 'webmock'
|
25
26
|
end
|
@@ -5,6 +5,7 @@
|
|
5
5
|
|
6
6
|
require "logstash/devutils/rspec/spec_helper"
|
7
7
|
require "logstash/outputs/datadog_logs"
|
8
|
+
require 'webmock/rspec'
|
8
9
|
|
9
10
|
describe LogStash::Outputs::DatadogLogs do
|
10
11
|
context "should register" do
|
@@ -79,7 +80,7 @@ describe LogStash::Outputs::DatadogLogs do
|
|
79
80
|
end
|
80
81
|
|
81
82
|
it "should truncate events whose length is bigger than the max request size" do
|
82
|
-
input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}),"foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
|
83
|
+
input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}), "foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
|
83
84
|
actual_events = subject.batch_http_events(input_events, 10, 3)
|
84
85
|
expect(actual_events.length).to eq(3)
|
85
86
|
expect(actual_events[0][0]).to eq("dd1")
|
@@ -88,6 +89,80 @@ describe LogStash::Outputs::DatadogLogs do
|
|
88
89
|
end
|
89
90
|
end
|
90
91
|
|
92
|
+
context "when facing HTTP connection issues" do
|
93
|
+
it "should retry when server is returning 5XX" do
|
94
|
+
api_key = 'XXX'
|
95
|
+
stub_dd_request_with_return_code(api_key, 500)
|
96
|
+
payload = '{}'
|
97
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
98
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should not retry when server is returning 4XX" do
|
102
|
+
api_key = 'XXX'
|
103
|
+
stub_dd_request_with_return_code(api_key, 400)
|
104
|
+
payload = '{}'
|
105
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
106
|
+
expect { client.send(payload) }.to_not raise_error
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should retry when facing a timeout exception from manticore" do
|
110
|
+
api_key = 'XXX'
|
111
|
+
stub_dd_request_with_error(api_key, Manticore::Timeout)
|
112
|
+
payload = '{}'
|
113
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
114
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should retry when facing a socket exception from manticore" do
|
118
|
+
api_key = 'XXX'
|
119
|
+
stub_dd_request_with_error(api_key, Manticore::SocketException)
|
120
|
+
payload = '{}'
|
121
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
122
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
123
|
+
end
|
124
|
+
|
125
|
+
it "should retry when facing a client protocol exception from manticore" do
|
126
|
+
api_key = 'XXX'
|
127
|
+
stub_dd_request_with_error(api_key, Manticore::ClientProtocolException)
|
128
|
+
payload = '{}'
|
129
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
130
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
131
|
+
end
|
132
|
+
|
133
|
+
it "should retry when facing a dns failure from manticore" do
|
134
|
+
api_key = 'XXX'
|
135
|
+
stub_dd_request_with_error(api_key, Manticore::ResolutionFailure)
|
136
|
+
payload = '{}'
|
137
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
138
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
139
|
+
end
|
140
|
+
|
141
|
+
it "should retry when facing a socket timeout from manticore" do
|
142
|
+
api_key = 'XXX'
|
143
|
+
stub_dd_request_with_error(api_key, Manticore::SocketTimeout)
|
144
|
+
payload = '{}'
|
145
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
146
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
147
|
+
end
|
148
|
+
|
149
|
+
it "should not retry when facing any other general error" do
|
150
|
+
api_key = 'XXX'
|
151
|
+
stub_dd_request_with_error(api_key, StandardError)
|
152
|
+
payload = '{}'
|
153
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
154
|
+
expect { client.send(payload) }.to raise_error(StandardError)
|
155
|
+
end
|
156
|
+
|
157
|
+
it "should not stop the forwarder when facing any client uncaught exception" do
|
158
|
+
api_key = 'XXX'
|
159
|
+
stub_dd_request_with_error(api_key, StandardError)
|
160
|
+
payload = '{}'
|
161
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
|
162
|
+
expect { client.send_retries(payload, 2, 2) }.to_not raise_error
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
91
166
|
context "when using TCP" do
|
92
167
|
it "should re-encode events" do
|
93
168
|
input_event = "{message=dd}"
|
@@ -101,4 +176,25 @@ describe LogStash::Outputs::DatadogLogs do
|
|
101
176
|
expect(encoded_event).to eq("xxx {...TRUNCATED...")
|
102
177
|
end
|
103
178
|
end
|
179
|
+
|
180
|
+
def stub_dd_request_with_return_code(api_key, return_code)
|
181
|
+
stub_dd_request(api_key).
|
182
|
+
to_return(status: return_code, body: "", headers: {})
|
183
|
+
end
|
184
|
+
|
185
|
+
def stub_dd_request_with_error(api_key, error)
|
186
|
+
stub_dd_request(api_key).
|
187
|
+
to_raise(error)
|
188
|
+
end
|
189
|
+
|
190
|
+
def stub_dd_request(api_key)
|
191
|
+
stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
|
192
|
+
with(
|
193
|
+
body: "{}",
|
194
|
+
headers: {
|
195
|
+
'Connection' => 'Keep-Alive',
|
196
|
+
'Content-Type' => 'application/json'
|
197
|
+
})
|
198
|
+
end
|
199
|
+
|
104
200
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-datadog_logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Datadog
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-03-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,6 +73,20 @@ dependencies:
|
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
name: webmock
|
83
|
+
prerelease: false
|
84
|
+
type: :development
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
76
90
|
description:
|
77
91
|
email: support@datadoghq.com
|
78
92
|
executables: []
|