logstash-output-newrelic 1.3.0 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/DEVELOPER.md +13 -2
- data/Gemfile +12 -0
- data/lib/logstash/outputs/newrelic.rb +57 -37
- data/lib/logstash/outputs/newrelic_version/version.rb +1 -1
- data/spec/outputs/input_17997_messages_resulting_in_2680KB_compressed_payload.json +17997 -0
- data/spec/outputs/input_5000_messages_resulting_in_740KB_compressed_payload.json +5000 -0
- data/spec/outputs/newrelic_spec.rb +129 -27
- data/spec/outputs/single_input_message_exceeeding_1MB_once_compressed.json +1 -0
- metadata +7 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 25593f8efc45dce5d2366a19ce89a4d722f6e5a55fdb3d6d8cad607d551dcbd7
|
4
|
+
data.tar.gz: d729dc031d6276a9dafdd1ae5a7615bd378993437a36b6828c540fc901839c88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 82c061a523aab278574d91373f8691c1499d0560b04e96dd8a8f178b14438eaceb7236dc62cea8ca6ac15d56e9ffa7acdfc2534338d4ae00306b0ef3e47b9f58
|
7
|
+
data.tar.gz: a6b92e08674d6d67e04d9ca2cb5bc0007c717e43c07c888399f715e4c5de546a58cf8237ae9128b742d5cd96ba7b9d52142e11875d1e02b9649af7e515e5e3a0
|
data/DEVELOPER.md
CHANGED
@@ -2,12 +2,23 @@
|
|
2
2
|
|
3
3
|
# Getting started
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
**NOTE for Mac M1 users: ** note that `jruby-9.3.3.0` is the first jruby compatible with Mac M1 processors. In order to
|
6
|
+
develop the plugin locally, we recommend using this version. Nevertheless, when building the gem file (in the GH workflows),
|
7
|
+
we keep using jruby 9.2.13.0 in order to be backwards-compatible with older Logstash versions.
|
8
|
+
|
9
|
+
* Install RVM:
|
10
|
+
* `command curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -`
|
11
|
+
* `\curl -sSL https://get.rvm.io | bash -s stable`
|
12
|
+
* Reopen the terminal for `rvm` command to be available
|
13
|
+
* Install JRuby: `rvm install jruby-9.2.13.0`.
|
14
|
+
* Use that JRuby: `rvm use jruby-9.2.13.0`
|
15
|
+
* Ensure your terminal is using Java 11.
|
7
16
|
* Install Bundler gem: `jruby -S gem install bundler`
|
8
17
|
|
9
18
|
# Developing
|
10
19
|
|
20
|
+
* Ensure you have `logstash` installed locally (required for unit testing): `brew install logstash`
|
21
|
+
* Ensure your `logstash` path matches the one in `Gemfile`
|
11
22
|
* Install dependencies: `jruby -S bundle install`
|
12
23
|
* Write tests and production code!
|
13
24
|
* Bump version: edit version file `version.rb`
|
data/Gemfile
CHANGED
@@ -1,3 +1,15 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
2
|
gemspec
|
3
3
|
|
4
|
+
# The following is required to locally develop this plugin. Note that this Gemfile is NOT used when building the gem
|
5
|
+
# file for this plugin (see merge-to-master.yml), only when unit testing. When unit-testing, we need to have logstash-core
|
6
|
+
# in our local machine, given that the logstash-core GEM has not been published since 5.6.0.
|
7
|
+
# See: https://github.com/elastic/logstash/pull/14229 https://github.com/elastic/logstash/issues/14203
|
8
|
+
# And: https://github.com/elastic/logstash/pull/14229
|
9
|
+
|
10
|
+
logstash_path = ENV['LOGSTASH_PATH'] || '/opt/homebrew/Cellar/logstash/8.9.0/libexec'
|
11
|
+
|
12
|
+
if Dir.exist?(logstash_path)
|
13
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
14
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
15
|
+
end
|
@@ -12,9 +12,10 @@ require_relative './exception/error'
|
|
12
12
|
|
13
13
|
class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
14
14
|
java_import java.util.concurrent.Executors;
|
15
|
-
java_import java.util.concurrent.Semaphore;
|
16
15
|
|
17
|
-
|
16
|
+
RETRIABLE_CODES = Set[408, 429, 500, 502, 503, 504, 599]
|
17
|
+
|
18
|
+
MAX_PAYLOAD_SIZE_BYTES = 1_000_000
|
18
19
|
|
19
20
|
config_name "newrelic"
|
20
21
|
|
@@ -43,7 +44,6 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
43
44
|
'Content-Type' => 'application/json'
|
44
45
|
}.merge(auth).freeze
|
45
46
|
@executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
|
46
|
-
@semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
|
47
47
|
end
|
48
48
|
|
49
49
|
# Used by tests so that the test run can complete (background threads prevent JVM exit)
|
@@ -67,34 +67,40 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
67
67
|
end
|
68
68
|
end
|
69
69
|
|
70
|
-
def
|
71
|
-
|
72
|
-
|
73
|
-
:attributes => event_hash
|
74
|
-
}
|
70
|
+
def to_nr_logs(logstash_events)
|
71
|
+
logstash_events.map do |logstash_event|
|
72
|
+
event_hash = logstash_event.to_hash
|
75
73
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
end
|
81
|
-
if event_hash['timestamp']
|
82
|
-
log_message_hash['timestamp'] = event_hash['timestamp']
|
83
|
-
log_message_hash[:attributes].delete('timestamp')
|
84
|
-
end
|
74
|
+
nr_log_message_hash = {
|
75
|
+
# non-intrinsic attributes get put into 'attributes'
|
76
|
+
:attributes => event_hash
|
77
|
+
}
|
85
78
|
|
86
|
-
|
79
|
+
# intrinsic attributes go at the top level
|
80
|
+
if event_hash['message']
|
81
|
+
nr_log_message_hash['message'] = event_hash['message']
|
82
|
+
nr_log_message_hash[:attributes].delete('message')
|
83
|
+
end
|
84
|
+
if event_hash['timestamp']
|
85
|
+
nr_log_message_hash['timestamp'] = event_hash['timestamp']
|
86
|
+
nr_log_message_hash[:attributes].delete('timestamp')
|
87
|
+
end
|
88
|
+
|
89
|
+
nr_log_message_hash
|
90
|
+
end
|
87
91
|
end
|
88
92
|
|
89
|
-
def multi_receive(
|
90
|
-
if
|
93
|
+
def multi_receive(logstash_events)
|
94
|
+
if logstash_events.empty?
|
91
95
|
return
|
92
96
|
end
|
93
97
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
+
nr_logs = to_nr_logs(logstash_events)
|
99
|
+
|
100
|
+
package_and_send_recursively(nr_logs)
|
101
|
+
end
|
102
|
+
|
103
|
+
def package_and_send_recursively(nr_logs)
|
98
104
|
payload = {
|
99
105
|
:common => {
|
100
106
|
:attributes => {
|
@@ -104,19 +110,29 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
104
110
|
}
|
105
111
|
}
|
106
112
|
},
|
107
|
-
:logs =>
|
113
|
+
:logs => nr_logs
|
108
114
|
}
|
109
|
-
|
115
|
+
|
110
116
|
execute = @executor.java_method :submit, [java.lang.Runnable]
|
111
117
|
execute.call do
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
118
|
+
compressed_payload = StringIO.new
|
119
|
+
gzip = Zlib::GzipWriter.new(compressed_payload)
|
120
|
+
gzip << [payload].to_json
|
121
|
+
gzip.close
|
122
|
+
|
123
|
+
compressed_size = compressed_payload.string.bytesize
|
124
|
+
log_record_count = nr_logs.length
|
125
|
+
|
126
|
+
if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
|
127
|
+
@logger.error("Can't compress record below required maximum packet size and it will be discarded.")
|
128
|
+
elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
|
129
|
+
@logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
|
130
|
+
split_index = log_record_count / 2
|
131
|
+
package_and_send_recursively(nr_logs[0...split_index])
|
132
|
+
package_and_send_recursively(nr_logs[split_index..-1])
|
133
|
+
else
|
134
|
+
@logger.debug("Payload compressed size: #{compressed_size}")
|
135
|
+
nr_send(compressed_payload.string)
|
120
136
|
end
|
121
137
|
end
|
122
138
|
end
|
@@ -129,6 +145,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
129
145
|
|
130
146
|
def nr_send(payload)
|
131
147
|
retries = 0
|
148
|
+
retry_duration = 1
|
149
|
+
|
132
150
|
begin
|
133
151
|
http = Net::HTTP.new(@end_point.host, 443)
|
134
152
|
request = Net::HTTP::Post.new(@end_point.request_uri)
|
@@ -147,7 +165,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
147
165
|
@logger.error(e.message)
|
148
166
|
if (should_retry(retries) && is_retryable_code(e))
|
149
167
|
retries += 1
|
150
|
-
sleep(
|
168
|
+
sleep(retry_duration)
|
169
|
+
retry_duration *= 2
|
151
170
|
retry
|
152
171
|
end
|
153
172
|
rescue => e
|
@@ -162,7 +181,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
162
181
|
:error_class => e.class.name,
|
163
182
|
:backtrace => e.backtrace
|
164
183
|
)
|
165
|
-
sleep(
|
184
|
+
sleep(retry_duration)
|
185
|
+
retry_duration *= 2
|
166
186
|
retry
|
167
187
|
else
|
168
188
|
@logger.error(
|
@@ -181,6 +201,6 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
|
|
181
201
|
|
182
202
|
def is_retryable_code(response_error)
|
183
203
|
error_code = response_error.response_code
|
184
|
-
|
204
|
+
RETRIABLE_CODES.include?(error_code)
|
185
205
|
end
|
186
206
|
end # class LogStash::Outputs::NewRelic
|