logstash-output-scalyr 0.1.18.beta → 0.1.19.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +1 -1
- data/lib/logstash/outputs/scalyr.rb +2 -1
- data/lib/scalyr/common/util.rb +5 -3
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/logstash/outputs/scalyr_spec.rb +34 -0
- data/spec/scalyr/common/util_spec.rb +36 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '084d41a09e7ef86214868f26a97639169df8982d9488eb827d337ef53f02fc07'
|
4
|
+
data.tar.gz: 5bd3b17804c60044901d2f40c40419c119e680f7ac4a4ffc543701a885cb0c2e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 837bd2d4d142f31bcd30494175731818de9f60f9f39ebc13cf0ae92b36e58ad4b6d288ba2f92109323f096d30a1774334651907b07e1d1310f6b42dd24698b2e
|
7
|
+
data.tar.gz: 221e47feab585a99dc69b475f446f9a9b4f9d2bd91ffd60753886438c348e0139618afa46f3889180aba68582ed9532376ee5449fb0cf91bb05cbb796c11a08c
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.1.19.beta
|
4
|
+
- Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
|
5
|
+
by setting the `fix_deep_flattening_delimiters` configuration option to true.
|
6
|
+
|
3
7
|
## 0.1.18.beta
|
4
8
|
- Add metrics for successfully sent and failed logstash events, and retries.
|
5
9
|
- Make array flattening optional during nested value flattening with the `flatten_nested_arrays` configuration option.
|
data/README.md
CHANGED
@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
|
|
10
10
|
# Quick start
|
11
11
|
|
12
12
|
1. Build the gem, run `gem build logstash-output-scalyr.gemspec`
|
13
|
-
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.
|
13
|
+
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.19.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
|
14
14
|
3. Configure the output plugin (e.g. add it to a pipeline .conf)
|
15
15
|
4. Restart Logstash
|
16
16
|
|
@@ -70,6 +70,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
70
70
|
config :flatten_nested_values, :validate => :boolean, :default => false
|
71
71
|
config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
|
72
72
|
config :flatten_nested_arrays, :validate => :boolean, :default => true
|
73
|
+
config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
|
73
74
|
|
74
75
|
# If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
|
75
76
|
# :flat_tag_value
|
@@ -632,7 +633,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
632
633
|
# flatten record
|
633
634
|
if @flatten_nested_values
|
634
635
|
start_time = Time.now.to_f
|
635
|
-
record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays)
|
636
|
+
record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters)
|
636
637
|
end_time = Time.now.to_f
|
637
638
|
flatten_nested_values_duration = end_time - start_time
|
638
639
|
end
|
data/lib/scalyr/common/util.rb
CHANGED
@@ -4,7 +4,9 @@ module Scalyr; module Common; module Util;
|
|
4
4
|
# Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
|
5
5
|
# nested keys. Returned keys are always strings. If a non-hash or array is provided, raises TypeError.
|
6
6
|
# Please see rspec util_spec.rb for expected behavior.
|
7
|
-
|
7
|
+
# Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
|
8
|
+
# because some queries and dashboards already rely on the broken functionality.
|
9
|
+
def self.flatten(obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false)
|
8
10
|
|
9
11
|
# base case is input object is not enumerable, in which case simply return it
|
10
12
|
if !obj.respond_to?(:each)
|
@@ -20,7 +22,7 @@ def self.flatten(obj, delimiter='_', flatten_arrays=true)
|
|
20
22
|
# input object is a hash
|
21
23
|
obj.each do |key, value|
|
22
24
|
if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
|
23
|
-
flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
|
25
|
+
flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
|
24
26
|
result["#{key}#{delimiter}#{subkey}"] = subvalue
|
25
27
|
end
|
26
28
|
else
|
@@ -33,7 +35,7 @@ def self.flatten(obj, delimiter='_', flatten_arrays=true)
|
|
33
35
|
# input object is an array or set
|
34
36
|
obj.each_with_index do |value, index|
|
35
37
|
if value.respond_to?(:each)
|
36
|
-
flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
|
38
|
+
flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
|
37
39
|
result["#{index}#{delimiter}#{subkey}"] = subvalue
|
38
40
|
end
|
39
41
|
else
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
PLUGIN_VERSION = "v0.1.
|
2
|
+
PLUGIN_VERSION = "v0.1.19.beta"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.19.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -265,6 +265,40 @@ describe LogStash::Outputs::Scalyr do
|
|
265
265
|
'flatten_nested_values_delimiter' => ".",
|
266
266
|
}
|
267
267
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
268
|
+
it "flattens nested values with a period" do
|
269
|
+
allow(plugin).to receive(:send_status).and_return(nil)
|
270
|
+
plugin.register
|
271
|
+
result = plugin.build_multi_event_request_array(sample_events)
|
272
|
+
body = JSON.parse(result[0][:body])
|
273
|
+
expect(body['events'].size).to eq(3)
|
274
|
+
expect(body['events'][2]['attrs']).to eq({
|
275
|
+
"nested.a" => 1,
|
276
|
+
"nested.b_0" => 3,
|
277
|
+
"nested.b_1" => 4,
|
278
|
+
"nested.b_2" => 5,
|
279
|
+
'seq' => 3,
|
280
|
+
'source_file' => 'my file 3',
|
281
|
+
'source_host' => 'my host 3',
|
282
|
+
'serverHost' => 'Logstash',
|
283
|
+
"tag_prefix_t1" => "true",
|
284
|
+
"tag_prefix_t2" => "true",
|
285
|
+
"tag_prefix_t3" => "true",
|
286
|
+
"parser" => "logstashParser",
|
287
|
+
})
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
context "when configured to flatten values with custom delimiter and deep delimiter fix" do
|
292
|
+
config = {
|
293
|
+
'api_write_token' => '1234',
|
294
|
+
'flatten_tags' => true,
|
295
|
+
'flat_tag_value' => 'true',
|
296
|
+
'flat_tag_prefix' => 'tag_prefix_',
|
297
|
+
'flatten_nested_values' => true, # this converts into string 'true'
|
298
|
+
'flatten_nested_values_delimiter' => ".",
|
299
|
+
'fix_deep_flattening_delimiters' => true,
|
300
|
+
}
|
301
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
268
302
|
it "flattens nested values with a period" do
|
269
303
|
allow(plugin).to receive(:send_status).and_return(nil)
|
270
304
|
plugin.register
|
@@ -212,6 +212,42 @@ describe Scalyr::Common::Util do
|
|
212
212
|
expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
|
213
213
|
end
|
214
214
|
|
215
|
+
it "accepts custom delimiters with greater depth" do
|
216
|
+
din = {
|
217
|
+
'a' => 1,
|
218
|
+
'b' => {
|
219
|
+
'c' => {
|
220
|
+
'e' => 100
|
221
|
+
},
|
222
|
+
'd' => 200,
|
223
|
+
}
|
224
|
+
}
|
225
|
+
dout = {
|
226
|
+
'a' => 1,
|
227
|
+
'b:c_e' => 100,
|
228
|
+
'b:d' => 200,
|
229
|
+
}
|
230
|
+
expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
|
231
|
+
end
|
232
|
+
|
233
|
+
it "accepts custom delimiters with greater depth and deep delimiters fix" do
|
234
|
+
din = {
|
235
|
+
'a' => 1,
|
236
|
+
'b' => {
|
237
|
+
'c' => {
|
238
|
+
'e' => 100
|
239
|
+
},
|
240
|
+
'd' => 200,
|
241
|
+
}
|
242
|
+
}
|
243
|
+
dout = {
|
244
|
+
'a' => 1,
|
245
|
+
'b:c:e' => 100,
|
246
|
+
'b:d' => 200,
|
247
|
+
}
|
248
|
+
expect(Scalyr::Common::Util.flatten(din, ':', true, true)).to eq(dout)
|
249
|
+
end
|
250
|
+
|
215
251
|
it "stringifies non-string keys" do
|
216
252
|
din = {
|
217
253
|
'a' => 1,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.19.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-07-
|
11
|
+
date: 2021-07-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|