logstash-output-datadog_logs 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/lib/logstash/outputs/datadog_logs.rb +21 -7
- data/lib/logstash/outputs/version.rb +5 -0
- data/logstash-output-datadog_logs.gemspec +8 -2
- data/spec/outputs/datadog_logs_spec.rb +105 -83
- metadata +8 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 635ef3d038ba35b57528f006e81455e20b7cd27217a8fe6188f65d5229e13fe3
|
4
|
+
data.tar.gz: 6b4325a4cf44791c40aa18360ba7fd9c92a451fb94b182c5a0803de711e3d462
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e555d642d57cd65971b9e05c2cd6d76c8919ec7413d1dab24eb9fe6eb8d9be3ce51dcfd0dc4751d77f052d87f49ea032fb6ce488c6b99350ae31b41f93908d2e
|
7
|
+
data.tar.gz: 2b89082813230155ebc747da6e5073aaa7f74d0ad2b1c922e8c577f670af6a2306447a0384229a0382f2c499de1741b20d11b03bc89f32f314939fee269e17c3
|
data/CHANGELOG.md
CHANGED
@@ -8,6 +8,7 @@ require "logstash/outputs/base"
|
|
8
8
|
require "logstash/namespace"
|
9
9
|
require "zlib"
|
10
10
|
|
11
|
+
require_relative "version"
|
11
12
|
|
12
13
|
# DatadogLogs lets you send logs to Datadog
|
13
14
|
# based on LogStash events.
|
@@ -33,11 +34,12 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
33
34
|
config :use_compression, :validate => :boolean, :required => false, :default => true
|
34
35
|
config :compression_level, :validate => :number, :required => false, :default => 6
|
35
36
|
config :no_ssl_validation, :validate => :boolean, :required => false, :default => false
|
37
|
+
config :force_v1_routes, :validate => :boolean, :required => false, :default => false # force using deprecated v1 routes
|
36
38
|
|
37
39
|
# Register the plugin to logstash
|
38
40
|
public
|
39
41
|
def register
|
40
|
-
@client = new_client(@logger, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @port, @use_compression)
|
42
|
+
@client = new_client(@logger, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @port, @use_compression, @force_v1_routes)
|
41
43
|
end
|
42
44
|
|
43
45
|
# Logstash shutdown hook
|
@@ -143,9 +145,9 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
143
145
|
end
|
144
146
|
|
145
147
|
# Build a new transport client
|
146
|
-
def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression)
|
148
|
+
def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression, force_v1_routes)
|
147
149
|
if use_http
|
148
|
-
DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key
|
150
|
+
DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes
|
149
151
|
else
|
150
152
|
DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, port
|
151
153
|
end
|
@@ -192,15 +194,26 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
192
194
|
::Manticore::ResolutionFailure
|
193
195
|
]
|
194
196
|
|
195
|
-
def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
|
197
|
+
def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes)
|
196
198
|
@logger = logger
|
197
199
|
protocol = use_ssl ? "https" : "http"
|
198
|
-
|
200
|
+
|
199
201
|
@headers = {"Content-Type" => "application/json"}
|
200
202
|
if use_compression
|
201
203
|
@headers["Content-Encoding"] = "gzip"
|
202
204
|
end
|
203
|
-
|
205
|
+
|
206
|
+
if force_v1_routes
|
207
|
+
@url = "#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}"
|
208
|
+
else
|
209
|
+
@url = "#{protocol}://#{host}:#{port.to_s}/api/v2/logs"
|
210
|
+
@headers["DD-API-KEY"] = api_key
|
211
|
+
@headers["DD-EVP-ORIGIN"] = "logstash"
|
212
|
+
@headers["DD-EVP-ORIGIN-VERSION"] = DatadogLogStashPlugin::VERSION
|
213
|
+
end
|
214
|
+
|
215
|
+
logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled") + (force_v1_routes ? " using v1 routes" : " using v2 routes"))
|
216
|
+
|
204
217
|
config = {}
|
205
218
|
config[:ssl][:verify] = :disable if no_ssl_validation
|
206
219
|
@client = Manticore::Client.new(config)
|
@@ -209,7 +222,8 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
|
|
209
222
|
def send(payload)
|
210
223
|
begin
|
211
224
|
response = @client.post(@url, :body => payload, :headers => @headers).call
|
212
|
-
|
225
|
+
# in case of error or 429, we will retry sending this payload
|
226
|
+
if response.code >= 500 || response.code == 429
|
213
227
|
raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
|
214
228
|
end
|
215
229
|
if response.code >= 400
|
@@ -1,6 +1,12 @@
|
|
1
|
+
# Load version.rb containing the DatadogLogStashPlugin::VERSION
|
2
|
+
# for current Gem version.
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require "logstash/outputs/version.rb"
|
6
|
+
|
1
7
|
Gem::Specification.new do |s|
|
2
8
|
s.name = 'logstash-output-datadog_logs'
|
3
|
-
s.version =
|
9
|
+
s.version = DatadogLogStashPlugin::VERSION
|
4
10
|
s.licenses = ['Apache-2.0']
|
5
11
|
s.summary = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
|
6
12
|
s.homepage = 'https://www.datadoghq.com/'
|
@@ -21,6 +27,6 @@ Gem::Specification.new do |s|
|
|
21
27
|
s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
|
22
28
|
s.add_runtime_dependency 'logstash-codec-json'
|
23
29
|
|
24
|
-
s.add_development_dependency 'logstash-devutils'
|
30
|
+
s.add_development_dependency 'logstash-devutils', "= 1.3.6"
|
25
31
|
s.add_development_dependency 'webmock'
|
26
32
|
end
|
@@ -90,76 +90,86 @@ describe LogStash::Outputs::DatadogLogs do
|
|
90
90
|
end
|
91
91
|
|
92
92
|
context "when facing HTTP connection issues" do
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
93
|
+
[true, false].each do |force_v1_routes|
|
94
|
+
it "should retry when server is returning 5XX " + (force_v1_routes ? "using v1 routes" : "using v2 routes") do
|
95
|
+
api_key = 'XXX'
|
96
|
+
stub_dd_request_with_return_code(api_key, 500, force_v1_routes)
|
97
|
+
payload = '{}'
|
98
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
99
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should not retry when server is returning 4XX" do
|
103
|
+
api_key = 'XXX'
|
104
|
+
stub_dd_request_with_return_code(api_key, 400, force_v1_routes)
|
105
|
+
payload = '{}'
|
106
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
107
|
+
expect { client.send(payload) }.to_not raise_error
|
108
|
+
end
|
109
|
+
|
110
|
+
it "should retry when server is returning 429" do
|
111
|
+
api_key = 'XXX'
|
112
|
+
stub_dd_request_with_return_code(api_key, 429, force_v1_routes)
|
113
|
+
payload = '{}'
|
114
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
115
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should retry when facing a timeout exception from manticore" do
|
119
|
+
api_key = 'XXX'
|
120
|
+
stub_dd_request_with_error(api_key, Manticore::Timeout, force_v1_routes)
|
121
|
+
payload = '{}'
|
122
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
123
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
124
|
+
end
|
125
|
+
|
126
|
+
it "should retry when facing a socket exception from manticore" do
|
127
|
+
api_key = 'XXX'
|
128
|
+
stub_dd_request_with_error(api_key, Manticore::SocketException, force_v1_routes)
|
129
|
+
payload = '{}'
|
130
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
131
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
132
|
+
end
|
133
|
+
|
134
|
+
it "should retry when facing a client protocol exception from manticore" do
|
135
|
+
api_key = 'XXX'
|
136
|
+
stub_dd_request_with_error(api_key, Manticore::ClientProtocolException, force_v1_routes)
|
137
|
+
payload = '{}'
|
138
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
139
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
140
|
+
end
|
141
|
+
|
142
|
+
it "should retry when facing a dns failure from manticore" do
|
143
|
+
api_key = 'XXX'
|
144
|
+
stub_dd_request_with_error(api_key, Manticore::ResolutionFailure, force_v1_routes)
|
145
|
+
payload = '{}'
|
146
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
147
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
148
|
+
end
|
149
|
+
|
150
|
+
it "should retry when facing a socket timeout from manticore" do
|
151
|
+
api_key = 'XXX'
|
152
|
+
stub_dd_request_with_error(api_key, Manticore::SocketTimeout, force_v1_routes)
|
153
|
+
payload = '{}'
|
154
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
155
|
+
expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
|
156
|
+
end
|
157
|
+
|
158
|
+
it "should not retry when facing any other general error" do
|
159
|
+
api_key = 'XXX'
|
160
|
+
stub_dd_request_with_error(api_key, StandardError, force_v1_routes)
|
161
|
+
payload = '{}'
|
162
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
163
|
+
expect { client.send(payload) }.to raise_error(StandardError)
|
164
|
+
end
|
165
|
+
|
166
|
+
it "should not stop the forwarder when facing any client uncaught exception" do
|
167
|
+
api_key = 'XXX'
|
168
|
+
stub_dd_request_with_error(api_key, StandardError, force_v1_routes)
|
169
|
+
payload = '{}'
|
170
|
+
client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
|
171
|
+
expect { client.send_retries(payload, 2, 2) }.to_not raise_error
|
172
|
+
end
|
163
173
|
end
|
164
174
|
end
|
165
175
|
|
@@ -177,24 +187,36 @@ describe LogStash::Outputs::DatadogLogs do
|
|
177
187
|
end
|
178
188
|
end
|
179
189
|
|
180
|
-
def stub_dd_request_with_return_code(api_key, return_code)
|
181
|
-
stub_dd_request(api_key).
|
190
|
+
def stub_dd_request_with_return_code(api_key, return_code, force_v1_routes)
|
191
|
+
stub_dd_request(api_key, force_v1_routes).
|
182
192
|
to_return(status: return_code, body: "", headers: {})
|
183
193
|
end
|
184
194
|
|
185
|
-
def stub_dd_request_with_error(api_key, error)
|
186
|
-
stub_dd_request(api_key).
|
195
|
+
def stub_dd_request_with_error(api_key, error, force_v1_routes)
|
196
|
+
stub_dd_request(api_key, force_v1_routes).
|
187
197
|
to_raise(error)
|
188
198
|
end
|
189
199
|
|
190
|
-
def stub_dd_request(api_key)
|
191
|
-
|
200
|
+
def stub_dd_request(api_key, force_v1_routes)
|
201
|
+
if force_v1_routes
|
202
|
+
stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
|
203
|
+
with(
|
204
|
+
body: "{}",
|
205
|
+
headers: {
|
206
|
+
'Connection' => 'Keep-Alive',
|
207
|
+
'Content-Type' => 'application/json'
|
208
|
+
})
|
209
|
+
else
|
210
|
+
stub_request(:post, "http://datadog.com/api/v2/logs").
|
192
211
|
with(
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
}
|
212
|
+
body: "{}",
|
213
|
+
headers: {
|
214
|
+
'Connection' => 'Keep-Alive',
|
215
|
+
'Content-Type' => 'application/json',
|
216
|
+
'DD-API-KEY' => "#{api_key}",
|
217
|
+
'DD-EVP-ORIGIN' => 'logstash',
|
218
|
+
'DD-EVP-ORIGIN-VERSION' => DatadogLogStashPlugin::VERSION
|
219
|
+
})
|
220
|
+
end
|
198
221
|
end
|
199
|
-
|
200
|
-
end
|
222
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-datadog_logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Datadog
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-04-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -62,17 +62,17 @@ dependencies:
|
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
64
64
|
requirements:
|
65
|
-
- -
|
65
|
+
- - '='
|
66
66
|
- !ruby/object:Gem::Version
|
67
|
-
version:
|
67
|
+
version: 1.3.6
|
68
68
|
name: logstash-devutils
|
69
69
|
prerelease: false
|
70
70
|
type: :development
|
71
71
|
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - '='
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: 1.3.6
|
76
76
|
- !ruby/object:Gem::Dependency
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
@@ -100,6 +100,7 @@ files:
|
|
100
100
|
- NOTICE.TXT
|
101
101
|
- README.md
|
102
102
|
- lib/logstash/outputs/datadog_logs.rb
|
103
|
+
- lib/logstash/outputs/version.rb
|
103
104
|
- logstash-output-datadog_logs.gemspec
|
104
105
|
- spec/outputs/datadog_logs_spec.rb
|
105
106
|
homepage: https://www.datadoghq.com/
|
@@ -124,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
124
125
|
version: '0'
|
125
126
|
requirements: []
|
126
127
|
rubyforge_project:
|
127
|
-
rubygems_version: 2.7.
|
128
|
+
rubygems_version: 2.7.6
|
128
129
|
signing_key:
|
129
130
|
specification_version: 4
|
130
131
|
summary: DatadogLogs lets you send logs to Datadog based on LogStash events.
|