logstash-output-datadog_logs 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8148c91e062cc8e1999b780d95f529d9cd1516664f7872a96da51e1656c2b71d
4
- data.tar.gz: 6fd0ddbf65b8240e1609c0c74a57f7ffcf50e7945cf2601484007725ff5465b2
3
+ metadata.gz: 635ef3d038ba35b57528f006e81455e20b7cd27217a8fe6188f65d5229e13fe3
4
+ data.tar.gz: 6b4325a4cf44791c40aa18360ba7fd9c92a451fb94b182c5a0803de711e3d462
5
5
  SHA512:
6
- metadata.gz: ce5a190b7c1550eb4edcec848aee451abd9814894da3e9c4302611c83c1e9414373086b69e60372bfde6cd6ad3da611146ac82413563b95c49721bdb0a024c68
7
- data.tar.gz: 33c1d8afda9da935ccb5f9ee43388f424ab7c8fefdb61b16ef9ac474f5d1c5834c053f911bfadce596291ea62148ff0f0a1beaf7bf56819ad433ac7fea92ae71
6
+ metadata.gz: e555d642d57cd65971b9e05c2cd6d76c8919ec7413d1dab24eb9fe6eb8d9be3ce51dcfd0dc4751d77f052d87f49ea032fb6ce488c6b99350ae31b41f93908d2e
7
+ data.tar.gz: 2b89082813230155ebc747da6e5073aaa7f74d0ad2b1c922e8c577f670af6a2306447a0384229a0382f2c499de1741b20d11b03bc89f32f314939fee269e17c3
data/CHANGELOG.md CHANGED
@@ -1,5 +1,8 @@
1
+ ## 0.5.0
2
+ - Support Datadog v2 endpoints #28
3
+
1
4
  ## 0.4.1
2
- - Fix HTTP bug when remote server is timing out
5
+ - Fix HTTP bug when remote server is timing out
3
6
 
4
7
  ## 0.4.0
5
8
  - Enable HTTP forwarding for logs
@@ -8,6 +8,7 @@ require "logstash/outputs/base"
8
8
  require "logstash/namespace"
9
9
  require "zlib"
10
10
 
11
+ require_relative "version"
11
12
 
12
13
  # DatadogLogs lets you send logs to Datadog
13
14
  # based on LogStash events.
@@ -33,11 +34,12 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
33
34
  config :use_compression, :validate => :boolean, :required => false, :default => true
34
35
  config :compression_level, :validate => :number, :required => false, :default => 6
35
36
  config :no_ssl_validation, :validate => :boolean, :required => false, :default => false
37
+ config :force_v1_routes, :validate => :boolean, :required => false, :default => false # force using deprecated v1 routes
36
38
 
37
39
  # Register the plugin to logstash
38
40
  public
39
41
  def register
40
- @client = new_client(@logger, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @port, @use_compression)
42
+ @client = new_client(@logger, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @port, @use_compression, @force_v1_routes)
41
43
  end
42
44
 
43
45
  # Logstash shutdown hook
@@ -143,9 +145,9 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
143
145
  end
144
146
 
145
147
  # Build a new transport client
146
- def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression)
148
+ def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression, force_v1_routes)
147
149
  if use_http
148
- DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key
150
+ DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes
149
151
  else
150
152
  DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, port
151
153
  end
@@ -192,15 +194,26 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
192
194
  ::Manticore::ResolutionFailure
193
195
  ]
194
196
 
195
- def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
197
+ def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes)
196
198
  @logger = logger
197
199
  protocol = use_ssl ? "https" : "http"
198
- @url = "#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}"
200
+
199
201
  @headers = {"Content-Type" => "application/json"}
200
202
  if use_compression
201
203
  @headers["Content-Encoding"] = "gzip"
202
204
  end
203
- logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled"))
205
+
206
+ if force_v1_routes
207
+ @url = "#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}"
208
+ else
209
+ @url = "#{protocol}://#{host}:#{port.to_s}/api/v2/logs"
210
+ @headers["DD-API-KEY"] = api_key
211
+ @headers["DD-EVP-ORIGIN"] = "logstash"
212
+ @headers["DD-EVP-ORIGIN-VERSION"] = DatadogLogStashPlugin::VERSION
213
+ end
214
+
215
+ logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled") + (force_v1_routes ? " using v1 routes" : " using v2 routes"))
216
+
204
217
  config = {}
205
218
  config[:ssl][:verify] = :disable if no_ssl_validation
206
219
  @client = Manticore::Client.new(config)
@@ -209,7 +222,8 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
209
222
  def send(payload)
210
223
  begin
211
224
  response = @client.post(@url, :body => payload, :headers => @headers).call
212
- if response.code >= 500
225
+ # in case of error or 429, we will retry sending this payload
226
+ if response.code >= 500 || response.code == 429
213
227
  raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
214
228
  end
215
229
  if response.code >= 400
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatadogLogStashPlugin
4
+ VERSION = '0.5.0'
5
+ end
@@ -1,6 +1,12 @@
1
+ # Load version.rb containing the DatadogLogStashPlugin::VERSION
2
+ # for current Gem version.
3
+ lib = File.expand_path('../lib', __FILE__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+ require "logstash/outputs/version.rb"
6
+
1
7
  Gem::Specification.new do |s|
2
8
  s.name = 'logstash-output-datadog_logs'
3
- s.version = '0.4.1'
9
+ s.version = DatadogLogStashPlugin::VERSION
4
10
  s.licenses = ['Apache-2.0']
5
11
  s.summary = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
6
12
  s.homepage = 'https://www.datadoghq.com/'
@@ -21,6 +27,6 @@ Gem::Specification.new do |s|
21
27
  s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
22
28
  s.add_runtime_dependency 'logstash-codec-json'
23
29
 
24
- s.add_development_dependency 'logstash-devutils'
30
+ s.add_development_dependency 'logstash-devutils', "= 1.3.6"
25
31
  s.add_development_dependency 'webmock'
26
32
  end
@@ -90,76 +90,86 @@ describe LogStash::Outputs::DatadogLogs do
90
90
  end
91
91
 
92
92
  context "when facing HTTP connection issues" do
93
- it "should retry when server is returning 5XX" do
94
- api_key = 'XXX'
95
- stub_dd_request_with_return_code(api_key, 500)
96
- payload = '{}'
97
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
98
- expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
99
- end
100
-
101
- it "should not retry when server is returning 4XX" do
102
- api_key = 'XXX'
103
- stub_dd_request_with_return_code(api_key, 400)
104
- payload = '{}'
105
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
106
- expect { client.send(payload) }.to_not raise_error
107
- end
108
-
109
- it "should retry when facing a timeout exception from manticore" do
110
- api_key = 'XXX'
111
- stub_dd_request_with_error(api_key, Manticore::Timeout)
112
- payload = '{}'
113
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
114
- expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
115
- end
116
-
117
- it "should retry when facing a socket exception from manticore" do
118
- api_key = 'XXX'
119
- stub_dd_request_with_error(api_key, Manticore::SocketException)
120
- payload = '{}'
121
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
122
- expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
123
- end
124
-
125
- it "should retry when facing a client protocol exception from manticore" do
126
- api_key = 'XXX'
127
- stub_dd_request_with_error(api_key, Manticore::ClientProtocolException)
128
- payload = '{}'
129
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
130
- expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
131
- end
132
-
133
- it "should retry when facing a dns failure from manticore" do
134
- api_key = 'XXX'
135
- stub_dd_request_with_error(api_key, Manticore::ResolutionFailure)
136
- payload = '{}'
137
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
138
- expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
139
- end
140
-
141
- it "should retry when facing a socket timeout from manticore" do
142
- api_key = 'XXX'
143
- stub_dd_request_with_error(api_key, Manticore::SocketTimeout)
144
- payload = '{}'
145
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
146
- expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
147
- end
148
-
149
- it "should not retry when facing any other general error" do
150
- api_key = 'XXX'
151
- stub_dd_request_with_error(api_key, StandardError)
152
- payload = '{}'
153
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
154
- expect { client.send(payload) }.to raise_error(StandardError)
155
- end
156
-
157
- it "should not stop the forwarder when facing any client uncaught exception" do
158
- api_key = 'XXX'
159
- stub_dd_request_with_error(api_key, StandardError)
160
- payload = '{}'
161
- client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
162
- expect { client.send_retries(payload, 2, 2) }.to_not raise_error
93
+ [true, false].each do |force_v1_routes|
94
+ it "should retry when server is returning 5XX " + (force_v1_routes ? "using v1 routes" : "using v2 routes") do
95
+ api_key = 'XXX'
96
+ stub_dd_request_with_return_code(api_key, 500, force_v1_routes)
97
+ payload = '{}'
98
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
99
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
100
+ end
101
+
102
+ it "should not retry when server is returning 4XX" do
103
+ api_key = 'XXX'
104
+ stub_dd_request_with_return_code(api_key, 400, force_v1_routes)
105
+ payload = '{}'
106
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
107
+ expect { client.send(payload) }.to_not raise_error
108
+ end
109
+
110
+ it "should retry when server is returning 429" do
111
+ api_key = 'XXX'
112
+ stub_dd_request_with_return_code(api_key, 429, force_v1_routes)
113
+ payload = '{}'
114
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
115
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
116
+ end
117
+
118
+ it "should retry when facing a timeout exception from manticore" do
119
+ api_key = 'XXX'
120
+ stub_dd_request_with_error(api_key, Manticore::Timeout, force_v1_routes)
121
+ payload = '{}'
122
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
123
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
124
+ end
125
+
126
+ it "should retry when facing a socket exception from manticore" do
127
+ api_key = 'XXX'
128
+ stub_dd_request_with_error(api_key, Manticore::SocketException, force_v1_routes)
129
+ payload = '{}'
130
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
131
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
132
+ end
133
+
134
+ it "should retry when facing a client protocol exception from manticore" do
135
+ api_key = 'XXX'
136
+ stub_dd_request_with_error(api_key, Manticore::ClientProtocolException, force_v1_routes)
137
+ payload = '{}'
138
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
139
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
140
+ end
141
+
142
+ it "should retry when facing a dns failure from manticore" do
143
+ api_key = 'XXX'
144
+ stub_dd_request_with_error(api_key, Manticore::ResolutionFailure, force_v1_routes)
145
+ payload = '{}'
146
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
147
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
148
+ end
149
+
150
+ it "should retry when facing a socket timeout from manticore" do
151
+ api_key = 'XXX'
152
+ stub_dd_request_with_error(api_key, Manticore::SocketTimeout, force_v1_routes)
153
+ payload = '{}'
154
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
155
+ expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
156
+ end
157
+
158
+ it "should not retry when facing any other general error" do
159
+ api_key = 'XXX'
160
+ stub_dd_request_with_error(api_key, StandardError, force_v1_routes)
161
+ payload = '{}'
162
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
163
+ expect { client.send(payload) }.to raise_error(StandardError)
164
+ end
165
+
166
+ it "should not stop the forwarder when facing any client uncaught exception" do
167
+ api_key = 'XXX'
168
+ stub_dd_request_with_error(api_key, StandardError, force_v1_routes)
169
+ payload = '{}'
170
+ client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
171
+ expect { client.send_retries(payload, 2, 2) }.to_not raise_error
172
+ end
163
173
  end
164
174
  end
165
175
 
@@ -177,24 +187,36 @@ describe LogStash::Outputs::DatadogLogs do
177
187
  end
178
188
  end
179
189
 
180
- def stub_dd_request_with_return_code(api_key, return_code)
181
- stub_dd_request(api_key).
190
+ def stub_dd_request_with_return_code(api_key, return_code, force_v1_routes)
191
+ stub_dd_request(api_key, force_v1_routes).
182
192
  to_return(status: return_code, body: "", headers: {})
183
193
  end
184
194
 
185
- def stub_dd_request_with_error(api_key, error)
186
- stub_dd_request(api_key).
195
+ def stub_dd_request_with_error(api_key, error, force_v1_routes)
196
+ stub_dd_request(api_key, force_v1_routes).
187
197
  to_raise(error)
188
198
  end
189
199
 
190
- def stub_dd_request(api_key)
191
- stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
200
+ def stub_dd_request(api_key, force_v1_routes)
201
+ if force_v1_routes
202
+ stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
203
+ with(
204
+ body: "{}",
205
+ headers: {
206
+ 'Connection' => 'Keep-Alive',
207
+ 'Content-Type' => 'application/json'
208
+ })
209
+ else
210
+ stub_request(:post, "http://datadog.com/api/v2/logs").
192
211
  with(
193
- body: "{}",
194
- headers: {
195
- 'Connection' => 'Keep-Alive',
196
- 'Content-Type' => 'application/json'
197
- })
212
+ body: "{}",
213
+ headers: {
214
+ 'Connection' => 'Keep-Alive',
215
+ 'Content-Type' => 'application/json',
216
+ 'DD-API-KEY' => "#{api_key}",
217
+ 'DD-EVP-ORIGIN' => 'logstash',
218
+ 'DD-EVP-ORIGIN-VERSION' => DatadogLogStashPlugin::VERSION
219
+ })
220
+ end
198
221
  end
199
-
200
- end
222
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-datadog_logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Datadog
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-03-13 00:00:00.000000000 Z
12
+ date: 2022-04-25 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -62,17 +62,17 @@ dependencies:
62
62
  - !ruby/object:Gem::Dependency
63
63
  requirement: !ruby/object:Gem::Requirement
64
64
  requirements:
65
- - - ">="
65
+ - - '='
66
66
  - !ruby/object:Gem::Version
67
- version: '0'
67
+ version: 1.3.6
68
68
  name: logstash-devutils
69
69
  prerelease: false
70
70
  type: :development
71
71
  version_requirements: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ">="
73
+ - - '='
74
74
  - !ruby/object:Gem::Version
75
- version: '0'
75
+ version: 1.3.6
76
76
  - !ruby/object:Gem::Dependency
77
77
  requirement: !ruby/object:Gem::Requirement
78
78
  requirements:
@@ -100,6 +100,7 @@ files:
100
100
  - NOTICE.TXT
101
101
  - README.md
102
102
  - lib/logstash/outputs/datadog_logs.rb
103
+ - lib/logstash/outputs/version.rb
103
104
  - logstash-output-datadog_logs.gemspec
104
105
  - spec/outputs/datadog_logs_spec.rb
105
106
  homepage: https://www.datadoghq.com/
@@ -124,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
124
125
  version: '0'
125
126
  requirements: []
126
127
  rubyforge_project:
127
- rubygems_version: 2.7.10
128
+ rubygems_version: 2.7.6
128
129
  signing_key:
129
130
  specification_version: 4
130
131
  summary: DatadogLogs lets you send logs to Datadog based on LogStash events.