logstash-output-dynatrace 0.5.0 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +5 -5
- data/lib/logstash/outputs/dynatrace.rb +58 -7
- data/spec/outputs/dynatrace_spec.rb +59 -1
- data/version.yaml +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9281b9e498b9986e45d25eaadc74358f95547551baf97132483533997edf134d
|
4
|
+
data.tar.gz: 27bc3095ec54575becfaf75eb2b7bca7d186f95b2ab239713e83f8b9f58b363b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f4863a8fae10fe23afc85d4a6e31290ba3c0628d974c57c8cec6306adc2ba1da5c73905c7597c0954cb095ea1a24c6e182d028d8d7289f05951f77c214255b48
|
7
|
+
data.tar.gz: 53281ec7c6a7a8f6ac5dffae9cde5babe61d0a1b962d5c903e7e60975bb00d0528efc57c21c3b35d6f9bfc18b84c1549221c18dbe8ca3ae75e8cde764a94d7b7
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
## 0.6.0
|
2
|
+
|
3
|
+
- Disable cookie processing by default
|
4
|
+
|
5
|
+
## 0.5.1
|
6
|
+
|
7
|
+
- Split large batches into smaller requests in order to meet Dynatrace API payload size limitations
|
8
|
+
|
9
|
+
## 0.5.0
|
10
|
+
|
11
|
+
- Rewrite plugin using http client mixin
|
12
|
+
|
1
13
|
## 0.4.0
|
2
14
|
|
3
15
|
- Add` OpenSSL::SSL::SSLError` to the list of retried exception
|
data/README.md
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
+
<!-- omit in toc -->
|
1
2
|
# Logstash Dynatrace output plugin
|
2
3
|
|
3
|
-
[![Travis Build Status](https://
|
4
|
+
[![Travis Build Status](https://api.travis-ci.com/dynatrace-oss/logstash-output-dynatrace.svg?branch=main)](https://app.travis-ci.com/dynatrace-oss/logstash-output-dynatrace)
|
4
5
|
|
5
6
|
> This project is developed and maintained by Dynatrace R&D.
|
6
7
|
|
@@ -14,13 +15,12 @@
|
|
14
15
|
- [`ingest_endpoint_url`](#ingest_endpoint_url)
|
15
16
|
- [`api_key`](#api_key)
|
16
17
|
- [`ssl_verify_none`](#ssl_verify_none)
|
17
|
-
- [`codec`](#codec)
|
18
18
|
- [`enable_metric`](#enable_metric)
|
19
19
|
- [`id`](#id)
|
20
20
|
- [Troubleshooting issues](#troubleshooting-issues)
|
21
21
|
- [Enable Debug Logs](#enable-debug-logs)
|
22
22
|
|
23
|
-
A [Logstash](https://github.com/elastic/logstash) output plugin for sending logs to the Dynatrace [Generic log ingest API v2](https://
|
23
|
+
A [Logstash](https://github.com/elastic/logstash) output plugin for sending logs to the Dynatrace [Generic log ingest API v2](https://docs.dynatrace.com/docs/shortlink/api-log-monitoring-v2-post-ingest).
|
24
24
|
Please review the documentation for this API before using the plugin.
|
25
25
|
|
26
26
|
## Installation Prerequisites
|
@@ -81,7 +81,7 @@ The following configuration options are supported by all output plugins:
|
|
81
81
|
* Value type is [string](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string)
|
82
82
|
* Required
|
83
83
|
|
84
|
-
This is the full URL of the [Generic log ingest API v2](https://
|
84
|
+
This is the full URL of the [Generic log ingest API v2](https://docs.dynatrace.com/docs/shortlink/api-log-monitoring-v2-post-ingest/) endpoint on your ActiveGate.
|
85
85
|
Example: `"ingest_endpoint_url" => "https://abc123456.live.dynatrace.com/api/v2/logs/ingest"`
|
86
86
|
|
87
87
|
### `api_key`
|
@@ -89,7 +89,7 @@ Example: `"ingest_endpoint_url" => "https://abc123456.live.dynatrace.com/api/v2/
|
|
89
89
|
* Value type is [string](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string)
|
90
90
|
* Required
|
91
91
|
|
92
|
-
This is the [Dynatrace API token](https://
|
92
|
+
This is the [Dynatrace API token](https://docs.dynatrace.com/docs/shortlink/api-authentication) which will be used to authenticate log ingest requests.
|
93
93
|
It requires the `logs.ingest` (Ingest Logs) scope to be set and it is recommended to limit scope to only this one.
|
94
94
|
Example: `"api_key" => "dt0c01.4XLO3..."`
|
95
95
|
|
@@ -74,6 +74,12 @@ module LogStash
|
|
74
74
|
# Include body in debug logs when HTTP errors occur. Body may be large and include sensitive data.
|
75
75
|
config :debug_include_body, validate: :boolean, default: false
|
76
76
|
|
77
|
+
# Maximum size payload to send to the Dynatrace API in Bytes. Batches of events which would be larger than max_payload_size when serialized will be split into smaller batches of events.
|
78
|
+
config :max_payload_size, validate: :number, default: 4_500_000
|
79
|
+
|
80
|
+
# Disable cookie support. Overridden default value from LogStash::PluginMixins::HttpClient
|
81
|
+
config :cookies, :validate => :boolean, :default => false
|
82
|
+
|
77
83
|
def register
|
78
84
|
# ssl_verification_mode config is from mixin but ssl_verify_none is our documented config
|
79
85
|
@ssl_verification_mode = 'none' if @ssl_verify_none
|
@@ -103,6 +109,35 @@ module LogStash
|
|
103
109
|
end
|
104
110
|
end
|
105
111
|
|
112
|
+
class Batcher
|
113
|
+
def initialize(max_batch_size)
|
114
|
+
@max_batch_size = max_batch_size
|
115
|
+
@batch_events_size = 0
|
116
|
+
@serialized_events = []
|
117
|
+
end
|
118
|
+
|
119
|
+
def offer(serialized_event)
|
120
|
+
# 2 square brackets, the length of all previously serialized strings, commas, and the current event size
|
121
|
+
batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.length
|
122
|
+
return false if batch_size_bytes > @max_batch_size
|
123
|
+
|
124
|
+
@serialized_events.push(serialized_event)
|
125
|
+
@batch_events_size += serialized_event.length
|
126
|
+
true
|
127
|
+
end
|
128
|
+
|
129
|
+
def drain_and_serialize
|
130
|
+
out = "[#{@serialized_events.join(',')}]\n"
|
131
|
+
@batch_events_size = 0
|
132
|
+
@serialized_events = []
|
133
|
+
out
|
134
|
+
end
|
135
|
+
|
136
|
+
def empty?
|
137
|
+
@serialized_events.empty?
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
106
141
|
def make_headers
|
107
142
|
{
|
108
143
|
'User-Agent' => "logstash-output-dynatrace/#{DynatraceConstants::VERSION} logstash/#{LOGSTASH_VERSION}",
|
@@ -135,7 +170,24 @@ module LogStash
|
|
135
170
|
failures = java.util.concurrent.atomic.AtomicInteger.new(0)
|
136
171
|
|
137
172
|
pending = Queue.new
|
138
|
-
|
173
|
+
batcher = Batcher.new(@max_payload_size)
|
174
|
+
|
175
|
+
events.each do |event|
|
176
|
+
serialized_event = LogStash::Json.dump(event.to_hash)
|
177
|
+
if serialized_event.length > @max_payload_size
|
178
|
+
log_params = { size: serialized_event.length }
|
179
|
+
log_params[:body] = serialized_event if @debug_include_body
|
180
|
+
log_warning('Event larger than max_payload_size dropped', log_params)
|
181
|
+
next
|
182
|
+
end
|
183
|
+
|
184
|
+
next if batcher.offer(serialized_event)
|
185
|
+
|
186
|
+
pending << [batcher.drain_and_serialize, 0] unless batcher.empty?
|
187
|
+
batcher.offer(serialized_event)
|
188
|
+
end
|
189
|
+
|
190
|
+
pending << [batcher.drain_and_serialize, 0] unless batcher.empty?
|
139
191
|
|
140
192
|
while popped = pending.pop
|
141
193
|
break if popped == :done
|
@@ -199,11 +251,10 @@ module LogStash
|
|
199
251
|
end
|
200
252
|
|
201
253
|
def send_event(event, attempt)
|
202
|
-
body = event_body(event)
|
203
254
|
headers = make_headers
|
204
255
|
|
205
256
|
# Create an async request
|
206
|
-
response = client.post(ingest_endpoint_url, body:
|
257
|
+
response = client.post(ingest_endpoint_url, body: event, headers: headers)
|
207
258
|
|
208
259
|
if response_success?(response)
|
209
260
|
[:success, event, attempt]
|
@@ -231,7 +282,7 @@ module LogStash
|
|
231
282
|
end
|
232
283
|
if @debug_include_body
|
233
284
|
# body can be big and may have sensitive data
|
234
|
-
log_params[:body] =
|
285
|
+
log_params[:body] = event
|
235
286
|
end
|
236
287
|
end
|
237
288
|
log_failure('Could not fetch URL', log_params)
|
@@ -276,9 +327,9 @@ module LogStash
|
|
276
327
|
@logger.error(message, opts)
|
277
328
|
end
|
278
329
|
|
279
|
-
#
|
280
|
-
def
|
281
|
-
|
330
|
+
# This is split into a separate method mostly to help testing
|
331
|
+
def log_warning(message, opts)
|
332
|
+
@logger.warn(message, opts)
|
282
333
|
end
|
283
334
|
end
|
284
335
|
end
|
@@ -123,6 +123,7 @@ describe LogStash::Outputs::Dynatrace do
|
|
123
123
|
.with(ingest_endpoint_url, hash_including(:body, :headers))
|
124
124
|
.and_call_original
|
125
125
|
allow(subject).to receive(:log_failure).with(any_args)
|
126
|
+
allow(subject).to receive(:log_warning).with(any_args)
|
126
127
|
allow(subject).to receive(:log_retryable_response).with(any_args)
|
127
128
|
end
|
128
129
|
|
@@ -132,7 +133,7 @@ describe LogStash::Outputs::Dynatrace do
|
|
132
133
|
end
|
133
134
|
end
|
134
135
|
|
135
|
-
context 'performing a
|
136
|
+
context 'performing a request' do
|
136
137
|
describe 'invoking the request' do
|
137
138
|
before do
|
138
139
|
subject.multi_receive([event])
|
@@ -186,6 +187,63 @@ describe LogStash::Outputs::Dynatrace do
|
|
186
187
|
expect(subject).to have_received(:send_event).exactly(3).times
|
187
188
|
end
|
188
189
|
end
|
190
|
+
|
191
|
+
context 'with more than 4.5MB of events' do
|
192
|
+
before do
|
193
|
+
allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
|
194
|
+
subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 2_500_001 }) })
|
195
|
+
end
|
196
|
+
|
197
|
+
it 'should split the chunk into multiple requests' do
|
198
|
+
expect(subject).to have_received(:send_event).exactly(2).times
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
shared_examples('send small and drop large') do
|
203
|
+
it 'should only send the small event' do
|
204
|
+
expect(subject).to have_received(:send_event).exactly(1).times
|
205
|
+
end
|
206
|
+
|
207
|
+
it 'should log a warning' do
|
208
|
+
expect(subject).to have_received(:log_warning)
|
209
|
+
.with('Event larger than max_payload_size dropped', hash_including(:size))
|
210
|
+
.exactly(:once)
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
context 'with one small event and one too large event' do
|
215
|
+
before do
|
216
|
+
allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
|
217
|
+
subject.multi_receive([LogStash::Event.new({ 'event' => 'small' }),
|
218
|
+
LogStash::Event.new({ 'event' => 'n' * 4_500_001 })])
|
219
|
+
end
|
220
|
+
|
221
|
+
include_examples('send small and drop large')
|
222
|
+
end
|
223
|
+
|
224
|
+
context 'with one too large event and one small event' do
|
225
|
+
before do
|
226
|
+
allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
|
227
|
+
subject.multi_receive([LogStash::Event.new({ 'event' => 'n' * 4_500_001 }),
|
228
|
+
LogStash::Event.new({ 'event' => 'small' })])
|
229
|
+
end
|
230
|
+
|
231
|
+
include_examples('send small and drop large')
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
context 'max_payload_size 2MB' do
|
236
|
+
let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => 2_000_000 } }
|
237
|
+
subject { LogStash::Outputs::Dynatrace.new(config) }
|
238
|
+
|
239
|
+
before do
|
240
|
+
allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
|
241
|
+
subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 1_250_000 }) })
|
242
|
+
end
|
243
|
+
|
244
|
+
it 'should split the chunk into multiple requests' do
|
245
|
+
expect(subject).to have_received(:send_event).exactly(2).times
|
246
|
+
end
|
189
247
|
end
|
190
248
|
|
191
249
|
context 'on retryable unknown exception' do
|
data/version.yaml
CHANGED
@@ -1 +1 @@
|
|
1
|
-
logstash-output-dynatrace: '0.
|
1
|
+
logstash-output-dynatrace: '0.6.0'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-dynatrace
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dynatrace Open Source Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-02-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-codec-json
|