logstash-output-newrelic 1.1.2 → 1.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/DEVELOPER.md +26 -0
- data/README.md +1 -1
- data/lib/logstash/outputs/config/bigdecimal_patch.rb +24 -0
- data/lib/logstash/outputs/exception/error.rb +14 -0
- data/lib/logstash/outputs/newrelic.rb +70 -24
- data/lib/logstash/outputs/newrelic_version/version.rb +1 -1
- data/logstash-output-newrelic.gemspec +1 -1
- data/spec/outputs/newrelic_spec.rb +100 -1
- metadata +14 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '068a9ac838ba356ffc7cc6746933ad3970d5c72df54c6633e1c7453283d700c0'
|
4
|
+
data.tar.gz: 8f6557b7b8ab66e4d2de7d772cbf470403be8f125d2bb90d371c0af4581ec84e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c200662a9b6e2157193f9d21c7dadc9562db4ca0245525775742b5518afa73b7db801dc05b506c8a5c4df74e127c7c8e520a684b5a9cf709f29350c09c0b9ebc
|
7
|
+
data.tar.gz: b6f15e63a3a625fc86c4dc6423b37973a01ae8406f31aa944aac62c3343d7c7dcfb3a7642f54d0464c170b824f59126d5b25b87a459cea1c79f2d14e9fd087ab
|
data/DEVELOPER.md
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# Developing the plugin
|
2
|
+
|
3
|
+
# Getting started
|
4
|
+
|
5
|
+
* Install JRuby: `rbenv install jruby-9.2.5.0`
|
6
|
+
* Use that JRuby: `rbenv local jruby-9.2.5.0`
|
7
|
+
* Install Bundler gem: `jruby -S gem install bundler`
|
8
|
+
|
9
|
+
# Developing
|
10
|
+
|
11
|
+
* Install dependencies: `jruby -S bundle install`
|
12
|
+
* Write tests and production code!
|
13
|
+
* Bump version: edit version file `version.rb`
|
14
|
+
* Run tests: `jruby -S bundle exec rspec`
|
15
|
+
* Build the gem: `jruby -S gem build logstash-output-newrelic.gemspec`
|
16
|
+
|
17
|
+
# Testing it with a local Logstash install
|
18
|
+
|
19
|
+
Note: you may need to run the following commands outside of your checkout, since these should not
|
20
|
+
be run with the JRuby version that you've configured your checkout to use (by using rbenv).
|
21
|
+
|
22
|
+
* Remove previous version: `logstash-plugin remove logstash-output-newrelic`
|
23
|
+
* Add new version: `logstash-plugin install logstash-output-newrelic-<version>.gem`
|
24
|
+
* Restart logstash: For Homebrew: `brew services restart logstash`
|
25
|
+
* Cause a change that you've configured Logstash to pick up (for instance, append to a file you're having it monitor)
|
26
|
+
* Look in `https://one.newrelic.com/launcher/logger.log-launcher` for your log message
|
data/README.md
CHANGED
@@ -57,7 +57,7 @@ Exactly one of the following:
|
|
57
57
|
|---|---|---|
|
58
58
|
| concurrent_requests | The number of threads to make requests from | 1 |
|
59
59
|
| base_uri | New Relic ingestion endpoint | https://log-api.newrelic.com/log/v1 |
|
60
|
-
|
60
|
+
| max_retries | Maximum number attempts to retry to send a message. If set to 0, no re-attempts will be made. | 3 |
|
61
61
|
|
62
62
|
### EU plugin configuration
|
63
63
|
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'bigdecimal'
|
2
|
+
|
3
|
+
class BigDecimal
|
4
|
+
# Floating-point numbers that go through the 'json' Logstash filter get automatically converted into BigDecimals.
|
5
|
+
# Example of such a filter:
|
6
|
+
#
|
7
|
+
# filter {
|
8
|
+
# json {
|
9
|
+
# source => "message"
|
10
|
+
# }
|
11
|
+
# }
|
12
|
+
#
|
13
|
+
# The problem is that { "value" => BigDecimal('0.12345') } gets serialized into { "value": "0.12345e0"}. We do
|
14
|
+
# want to keep floating point numbers serialized as floating point numbers, even at the expense of loosing a little
|
15
|
+
# bit of precision during the conversion. So, in the above example, the correct serialization would be:
|
16
|
+
# { "value": 0.12345}
|
17
|
+
def to_json(options = nil) #:nodoc:
|
18
|
+
if finite?
|
19
|
+
self.to_f.to_s
|
20
|
+
else
|
21
|
+
'null'
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
class Error
|
2
|
+
class BadResponseCodeError < StandardError
|
3
|
+
attr_reader :url, :response_code
|
4
|
+
|
5
|
+
def initialize(response_code, url)
|
6
|
+
@response_code = response_code
|
7
|
+
@url = url
|
8
|
+
end
|
9
|
+
|
10
|
+
def message
|
11
|
+
"Got response code '#{@response_code}' contacting NewRelic at URL '#{@url}'"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -6,17 +6,23 @@ require 'uri'
|
|
6
6
|
require 'zlib'
|
7
7
|
require 'json'
|
8
8
|
require 'java'
|
9
|
+
require 'set'
|
10
|
+
require_relative './config/bigdecimal_patch'
|
11
|
+
require_relative './exception/error'
|
9
12
|
|
10
13
|
class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
11
14
|
java_import java.util.concurrent.Executors;
|
12
15
|
java_import java.util.concurrent.Semaphore;
|
13
16
|
|
17
|
+
NON_RETRYABLE_CODES = Set[401, 403]
|
18
|
+
|
14
19
|
config_name "newrelic"
|
15
20
|
|
16
21
|
config :api_key, :validate => :password, :required => false
|
17
22
|
config :license_key, :validate => :password, :required => false
|
18
23
|
config :concurrent_requests, :validate => :number, :default => 1
|
19
24
|
config :base_uri, :validate => :string, :default => "https://log-api.newrelic.com/log/v1"
|
25
|
+
config :max_retries, :validate => :number, :default => 3
|
20
26
|
|
21
27
|
public
|
22
28
|
|
@@ -26,12 +32,12 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
26
32
|
raise LogStash::ConfigurationError, "Must provide a license key or api key", caller
|
27
33
|
end
|
28
34
|
auth = {
|
29
|
-
@api_key.nil? ? 'X-License-Key': 'X-Insert-Key' =>
|
30
|
-
|
35
|
+
@api_key.nil? ? 'X-License-Key' : 'X-Insert-Key' =>
|
36
|
+
@api_key.nil? ? @license_key.value : @api_key.value
|
31
37
|
}
|
32
38
|
@header = {
|
33
|
-
|
34
|
-
|
39
|
+
'X-Event-Source' => 'logs',
|
40
|
+
'Content-Encoding' => 'gzip'
|
35
41
|
}.merge(auth).freeze
|
36
42
|
@executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
|
37
43
|
@semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
|
@@ -58,11 +64,10 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
58
64
|
end
|
59
65
|
end
|
60
66
|
|
61
|
-
|
62
67
|
def encode(event_hash)
|
63
68
|
log_message_hash = {
|
64
|
-
|
65
|
-
|
69
|
+
# non-intrinsic attributes get put into 'attributes'
|
70
|
+
:attributes => event_hash
|
66
71
|
}
|
67
72
|
|
68
73
|
# intrinsic attributes go at the top level
|
@@ -88,15 +93,15 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
88
93
|
payload.push(encode(event.to_hash))
|
89
94
|
end
|
90
95
|
payload = {
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
96
|
+
:common => {
|
97
|
+
:attributes => {
|
98
|
+
:plugin => {
|
99
|
+
:type => 'logstash',
|
100
|
+
:version => LogStash::Outputs::NewRelicVersion::VERSION,
|
101
|
+
}
|
102
|
+
}
|
103
|
+
},
|
104
|
+
:logs => payload
|
100
105
|
}
|
101
106
|
@semaphor.acquire()
|
102
107
|
execute = @executor.java_method :submit, [java.lang.Runnable]
|
@@ -115,17 +120,58 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
115
120
|
|
116
121
|
def handle_response(response)
|
117
122
|
if !(200 <= response.code.to_i && response.code.to_i < 300)
|
118
|
-
|
123
|
+
raise Error::BadResponseCodeError.new(response.code.to_i, @base_uri)
|
119
124
|
end
|
120
125
|
end
|
121
126
|
|
122
127
|
def nr_send(payload)
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
128
|
+
retries = 0
|
129
|
+
begin
|
130
|
+
http = Net::HTTP.new(@end_point.host, 443)
|
131
|
+
request = Net::HTTP::Post.new(@end_point.request_uri)
|
132
|
+
http.use_ssl = true
|
133
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
134
|
+
@header.each { |k, v| request[k] = v }
|
135
|
+
request.body = payload
|
136
|
+
handle_response(http.request(request))
|
137
|
+
rescue Error::BadResponseCodeError => e
|
138
|
+
@logger.error(e.message)
|
139
|
+
if (should_retry(retries) && is_retryable_code(e))
|
140
|
+
retries += 1
|
141
|
+
sleep(1)
|
142
|
+
retry
|
143
|
+
end
|
144
|
+
rescue => e
|
145
|
+
# Stuff that should never happen
|
146
|
+
# For all other errors print out full issues
|
147
|
+
if (should_retry(retries))
|
148
|
+
retries += 1
|
149
|
+
@logger.warn(
|
150
|
+
"An unknown error occurred sending a bulk request to NewRelic. Retrying...",
|
151
|
+
:retries => "attempt #{retries} of #{@max_retries}",
|
152
|
+
:error_message => e.message,
|
153
|
+
:error_class => e.class.name,
|
154
|
+
:backtrace => e.backtrace
|
155
|
+
)
|
156
|
+
sleep(1)
|
157
|
+
retry
|
158
|
+
else
|
159
|
+
@logger.error(
|
160
|
+
"An unknown error occurred sending a bulk request to NewRelic. Maximum of attempts reached, dropping logs.",
|
161
|
+
:error_message => e.message,
|
162
|
+
:error_class => e.class.name,
|
163
|
+
:backtrace => e.backtrace
|
164
|
+
)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def should_retry(retries)
|
170
|
+
retries < @max_retries
|
171
|
+
end
|
172
|
+
|
173
|
+
def is_retryable_code(response_error)
|
174
|
+
error_code = response_error.response_code
|
175
|
+
!NON_RETRYABLE_CODES.include?(error_code)
|
130
176
|
end
|
131
177
|
end # class LogStash::Outputs::NewRelic
|
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
|
|
7
7
|
s.name = 'logstash-output-newrelic'
|
8
8
|
s.version = LogStash::Outputs::NewRelicVersion::VERSION
|
9
9
|
s.licenses = ['Apache-2.0']
|
10
|
-
s.summary = "Sends
|
10
|
+
s.summary = "Sends Logstash events to New Relic"
|
11
11
|
s.homepage = 'https://github.com/newrelic/logstash-output-plugin'
|
12
12
|
s.authors = ['New Relic Logging Team']
|
13
13
|
s.email = 'logging-team@newrelic.com'
|
@@ -21,7 +21,6 @@ describe LogStash::Outputs::NewRelic do
|
|
21
21
|
}
|
22
22
|
}
|
23
23
|
|
24
|
-
|
25
24
|
before(:each) do
|
26
25
|
@newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(simple_config)
|
27
26
|
@newrelic_output.register
|
@@ -32,6 +31,7 @@ describe LogStash::Outputs::NewRelic do
|
|
32
31
|
@newrelic_output.shutdown
|
33
32
|
end
|
34
33
|
end
|
34
|
+
|
35
35
|
context "license key tests" do
|
36
36
|
it "sets license key when given in the header" do
|
37
37
|
stub_request(:any, base_uri).to_return(status: 200)
|
@@ -279,5 +279,104 @@ describe LogStash::Outputs::NewRelic do
|
|
279
279
|
.with { |request| single_gzipped_message(request.body)['message'] == 'Test message 2' })
|
280
280
|
.to have_been_made
|
281
281
|
end
|
282
|
+
|
283
|
+
it "retry when receive retryable http error code" do
|
284
|
+
stub_request(:any, base_uri)
|
285
|
+
.to_return(status: 500)
|
286
|
+
.to_return(status: 200)
|
287
|
+
|
288
|
+
event1 = LogStash::Event.new({ "message" => "Test message 1" })
|
289
|
+
@newrelic_output.multi_receive([event1])
|
290
|
+
|
291
|
+
wait_for(a_request(:post, base_uri)
|
292
|
+
.with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
|
293
|
+
.to have_been_made.times(2)
|
294
|
+
end
|
295
|
+
|
296
|
+
it "not retry when receive a non retryable http error code" do
|
297
|
+
stub_request(:any, base_uri)
|
298
|
+
.to_return(status: 401)
|
299
|
+
|
300
|
+
event1 = LogStash::Event.new({ "message" => "Test message 1" })
|
301
|
+
@newrelic_output.multi_receive([event1])
|
302
|
+
# Due the async behavior we need to wait to be sure that the method was not called more than 1 time
|
303
|
+
sleep(2)
|
304
|
+
wait_for(a_request(:post, base_uri)
|
305
|
+
.with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
|
306
|
+
.to have_been_made.times(1)
|
307
|
+
end
|
308
|
+
|
309
|
+
it "not retries when retry is disabled" do
|
310
|
+
@newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(
|
311
|
+
{ "base_uri" => base_uri, "license_key" => api_key, "max_retries" => '0' }
|
312
|
+
)
|
313
|
+
@newrelic_output.register
|
314
|
+
stub_request(:any, base_uri)
|
315
|
+
.to_return(status: 500)
|
316
|
+
|
317
|
+
event1 = LogStash::Event.new({ "message" => "Test message 1" })
|
318
|
+
@newrelic_output.multi_receive([event1])
|
319
|
+
# Due the async behavior we need to wait to be sure that the method was not called more than 1 time
|
320
|
+
sleep(2)
|
321
|
+
wait_for(a_request(:post, base_uri)
|
322
|
+
.with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
|
323
|
+
.to have_been_made.times(1)
|
324
|
+
end
|
325
|
+
|
326
|
+
it "retry when receive a not expected exception" do
|
327
|
+
stub_request(:any, base_uri)
|
328
|
+
.to_raise(StandardError.new("from test"))
|
329
|
+
.to_return(status: 200)
|
330
|
+
|
331
|
+
event1 = LogStash::Event.new({ "message" => "Test message 1" })
|
332
|
+
@newrelic_output.multi_receive([event1])
|
333
|
+
wait_for(a_request(:post, base_uri)
|
334
|
+
.with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
|
335
|
+
.to have_been_made.times(2)
|
336
|
+
end
|
337
|
+
end
|
338
|
+
|
339
|
+
context "JSON serialization" do
|
340
|
+
it "serializes floating point numbers as floating point numbers" do
|
341
|
+
stub_request(:any, base_uri).to_return(status: 200)
|
342
|
+
|
343
|
+
event = LogStash::Event.new({ "floatingpoint" => 0.12345 })
|
344
|
+
@newrelic_output.multi_receive([event])
|
345
|
+
|
346
|
+
wait_for(a_request(:post, base_uri)
|
347
|
+
.with { |request|
|
348
|
+
message = single_gzipped_message(request.body)
|
349
|
+
message['attributes']['floatingpoint'] == 0.12345
|
350
|
+
}
|
351
|
+
).to have_been_made
|
352
|
+
end
|
353
|
+
|
354
|
+
it "serializes BigDecimals as floating point numbers" do
|
355
|
+
stub_request(:any, base_uri).to_return(status: 200)
|
356
|
+
|
357
|
+
event = LogStash::Event.new({ "bigdecimal" => BigDecimal('0.12345') })
|
358
|
+
@newrelic_output.multi_receive([event])
|
359
|
+
|
360
|
+
wait_for(a_request(:post, base_uri)
|
361
|
+
.with { |request|
|
362
|
+
message = single_gzipped_message(request.body)
|
363
|
+
message['attributes']['bigdecimal'] == 0.12345
|
364
|
+
}
|
365
|
+
).to have_been_made
|
366
|
+
end
|
367
|
+
|
368
|
+
it "serializes NaN as null" do
|
369
|
+
stub_request(:any, base_uri).to_return(status: 200)
|
370
|
+
|
371
|
+
event = LogStash::Event.new({ "nan" => BigDecimal('NaN') })
|
372
|
+
@newrelic_output.multi_receive([event])
|
373
|
+
|
374
|
+
wait_for(a_request(:post, base_uri)
|
375
|
+
.with { |request|
|
376
|
+
message = single_gzipped_message(request.body)
|
377
|
+
message['attributes']['nan'] == nil
|
378
|
+
}
|
379
|
+
).to have_been_made
|
380
|
+
end
|
282
381
|
end
|
283
382
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-newrelic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- New Relic Logging Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -17,8 +17,8 @@ dependencies:
|
|
17
17
|
- !ruby/object:Gem::Version
|
18
18
|
version: '2.0'
|
19
19
|
name: logstash-core-plugin-api
|
20
|
-
prerelease: false
|
21
20
|
type: :runtime
|
21
|
+
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
@@ -31,8 +31,8 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '0'
|
33
33
|
name: logstash-codec-plain
|
34
|
-
prerelease: false
|
35
34
|
type: :runtime
|
35
|
+
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
@@ -45,8 +45,8 @@ dependencies:
|
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '0'
|
47
47
|
name: logstash-devutils
|
48
|
-
prerelease: false
|
49
48
|
type: :development
|
49
|
+
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
@@ -59,8 +59,8 @@ dependencies:
|
|
59
59
|
- !ruby/object:Gem::Version
|
60
60
|
version: '0'
|
61
61
|
name: webmock
|
62
|
-
prerelease: false
|
63
62
|
type: :development
|
63
|
+
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - ">="
|
@@ -73,8 +73,8 @@ dependencies:
|
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0'
|
75
75
|
name: rspec
|
76
|
-
prerelease: false
|
77
76
|
type: :development
|
77
|
+
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - ">="
|
@@ -87,8 +87,8 @@ dependencies:
|
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '0'
|
89
89
|
name: rspec-wait
|
90
|
-
prerelease: false
|
91
90
|
type: :development
|
91
|
+
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - ">="
|
@@ -101,8 +101,8 @@ dependencies:
|
|
101
101
|
- !ruby/object:Gem::Version
|
102
102
|
version: '0'
|
103
103
|
name: rspec_junit_formatter
|
104
|
-
prerelease: false
|
105
104
|
type: :development
|
105
|
+
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - ">="
|
@@ -116,9 +116,12 @@ extra_rdoc_files: []
|
|
116
116
|
files:
|
117
117
|
- CHANGELOG.md
|
118
118
|
- CONTRIBUTORS
|
119
|
+
- DEVELOPER.md
|
119
120
|
- Gemfile
|
120
121
|
- LICENSE
|
121
122
|
- README.md
|
123
|
+
- lib/logstash/outputs/config/bigdecimal_patch.rb
|
124
|
+
- lib/logstash/outputs/exception/error.rb
|
122
125
|
- lib/logstash/outputs/newrelic.rb
|
123
126
|
- lib/logstash/outputs/newrelic_version/version.rb
|
124
127
|
- logstash-output-newrelic.gemspec
|
@@ -144,10 +147,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
144
147
|
- !ruby/object:Gem::Version
|
145
148
|
version: '0'
|
146
149
|
requirements: []
|
147
|
-
|
148
|
-
rubygems_version: 2.7.6
|
150
|
+
rubygems_version: 3.0.6
|
149
151
|
signing_key:
|
150
152
|
specification_version: 4
|
151
|
-
summary: Sends
|
153
|
+
summary: Sends Logstash events to New Relic
|
152
154
|
test_files:
|
153
155
|
- spec/outputs/newrelic_spec.rb
|