logstash-output-scalyr 0.1.18.beta → 0.1.19.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +1 -1
- data/lib/logstash/outputs/scalyr.rb +2 -1
- data/lib/scalyr/common/util.rb +5 -3
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/logstash/outputs/scalyr_spec.rb +34 -0
- data/spec/scalyr/common/util_spec.rb +36 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '084d41a09e7ef86214868f26a97639169df8982d9488eb827d337ef53f02fc07'
|
4
|
+
data.tar.gz: 5bd3b17804c60044901d2f40c40419c119e680f7ac4a4ffc543701a885cb0c2e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 837bd2d4d142f31bcd30494175731818de9f60f9f39ebc13cf0ae92b36e58ad4b6d288ba2f92109323f096d30a1774334651907b07e1d1310f6b42dd24698b2e
|
7
|
+
data.tar.gz: 221e47feab585a99dc69b475f446f9a9b4f9d2bd91ffd60753886438c348e0139618afa46f3889180aba68582ed9532376ee5449fb0cf91bb05cbb796c11a08c
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.1.19.beta
|
4
|
+
- Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
|
5
|
+
by setting the `fix_deep_flattening_delimiters` configuration option to true.
|
6
|
+
|
3
7
|
## 0.1.18.beta
|
4
8
|
- Add metrics for successfully sent and failed logstash events, and retries.
|
5
9
|
- Make array flattening optional during nested value flattening with the `flatten_nested_arrays` configuration option.
|
data/README.md
CHANGED
@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
|
|
10
10
|
# Quick start
|
11
11
|
|
12
12
|
1. Build the gem, run `gem build logstash-output-scalyr.gemspec`
|
13
|
-
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.
|
13
|
+
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.19.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
|
14
14
|
3. Configure the output plugin (e.g. add it to a pipeline .conf)
|
15
15
|
4. Restart Logstash
|
16
16
|
|
@@ -70,6 +70,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
70
70
|
config :flatten_nested_values, :validate => :boolean, :default => false
|
71
71
|
config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
|
72
72
|
config :flatten_nested_arrays, :validate => :boolean, :default => true
|
73
|
+
config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
|
73
74
|
|
74
75
|
# If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
|
75
76
|
# :flat_tag_value
|
@@ -632,7 +633,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
632
633
|
# flatten record
|
633
634
|
if @flatten_nested_values
|
634
635
|
start_time = Time.now.to_f
|
635
|
-
record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays)
|
636
|
+
record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters)
|
636
637
|
end_time = Time.now.to_f
|
637
638
|
flatten_nested_values_duration = end_time - start_time
|
638
639
|
end
|
data/lib/scalyr/common/util.rb
CHANGED
@@ -4,7 +4,9 @@ module Scalyr; module Common; module Util;
|
|
4
4
|
# Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
|
5
5
|
# nested keys. Returned keys are always strings. If a non-hash or array is provided, raises TypeError.
|
6
6
|
# Please see rspec util_spec.rb for expected behavior.
|
7
|
-
|
7
|
+
# Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
|
8
|
+
# because some queries and dashboards already rely on the broken functionality.
|
9
|
+
def self.flatten(obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false)
|
8
10
|
|
9
11
|
# base case is input object is not enumerable, in which case simply return it
|
10
12
|
if !obj.respond_to?(:each)
|
@@ -20,7 +22,7 @@ def self.flatten(obj, delimiter='_', flatten_arrays=true)
|
|
20
22
|
# input object is a hash
|
21
23
|
obj.each do |key, value|
|
22
24
|
if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
|
23
|
-
flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
|
25
|
+
flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
|
24
26
|
result["#{key}#{delimiter}#{subkey}"] = subvalue
|
25
27
|
end
|
26
28
|
else
|
@@ -33,7 +35,7 @@ def self.flatten(obj, delimiter='_', flatten_arrays=true)
|
|
33
35
|
# input object is an array or set
|
34
36
|
obj.each_with_index do |value, index|
|
35
37
|
if value.respond_to?(:each)
|
36
|
-
flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
|
38
|
+
flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
|
37
39
|
result["#{index}#{delimiter}#{subkey}"] = subvalue
|
38
40
|
end
|
39
41
|
else
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
PLUGIN_VERSION = "v0.1.
|
2
|
+
PLUGIN_VERSION = "v0.1.19.beta"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.19.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -265,6 +265,40 @@ describe LogStash::Outputs::Scalyr do
|
|
265
265
|
'flatten_nested_values_delimiter' => ".",
|
266
266
|
}
|
267
267
|
plugin = LogStash::Outputs::Scalyr.new(config)
|
268
|
+
it "flattens nested values with a period" do
|
269
|
+
allow(plugin).to receive(:send_status).and_return(nil)
|
270
|
+
plugin.register
|
271
|
+
result = plugin.build_multi_event_request_array(sample_events)
|
272
|
+
body = JSON.parse(result[0][:body])
|
273
|
+
expect(body['events'].size).to eq(3)
|
274
|
+
expect(body['events'][2]['attrs']).to eq({
|
275
|
+
"nested.a" => 1,
|
276
|
+
"nested.b_0" => 3,
|
277
|
+
"nested.b_1" => 4,
|
278
|
+
"nested.b_2" => 5,
|
279
|
+
'seq' => 3,
|
280
|
+
'source_file' => 'my file 3',
|
281
|
+
'source_host' => 'my host 3',
|
282
|
+
'serverHost' => 'Logstash',
|
283
|
+
"tag_prefix_t1" => "true",
|
284
|
+
"tag_prefix_t2" => "true",
|
285
|
+
"tag_prefix_t3" => "true",
|
286
|
+
"parser" => "logstashParser",
|
287
|
+
})
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
context "when configured to flatten values with custom delimiter and deep delimiter fix" do
|
292
|
+
config = {
|
293
|
+
'api_write_token' => '1234',
|
294
|
+
'flatten_tags' => true,
|
295
|
+
'flat_tag_value' => 'true',
|
296
|
+
'flat_tag_prefix' => 'tag_prefix_',
|
297
|
+
'flatten_nested_values' => true, # this converts into string 'true'
|
298
|
+
'flatten_nested_values_delimiter' => ".",
|
299
|
+
'fix_deep_flattening_delimiters' => true,
|
300
|
+
}
|
301
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
268
302
|
it "flattens nested values with a period" do
|
269
303
|
allow(plugin).to receive(:send_status).and_return(nil)
|
270
304
|
plugin.register
|
@@ -212,6 +212,42 @@ describe Scalyr::Common::Util do
|
|
212
212
|
expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
|
213
213
|
end
|
214
214
|
|
215
|
+
it "accepts custom delimiters with greater depth" do
|
216
|
+
din = {
|
217
|
+
'a' => 1,
|
218
|
+
'b' => {
|
219
|
+
'c' => {
|
220
|
+
'e' => 100
|
221
|
+
},
|
222
|
+
'd' => 200,
|
223
|
+
}
|
224
|
+
}
|
225
|
+
dout = {
|
226
|
+
'a' => 1,
|
227
|
+
'b:c_e' => 100,
|
228
|
+
'b:d' => 200,
|
229
|
+
}
|
230
|
+
expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
|
231
|
+
end
|
232
|
+
|
233
|
+
it "accepts custom delimiters with greater depth and deep delimiters fix" do
|
234
|
+
din = {
|
235
|
+
'a' => 1,
|
236
|
+
'b' => {
|
237
|
+
'c' => {
|
238
|
+
'e' => 100
|
239
|
+
},
|
240
|
+
'd' => 200,
|
241
|
+
}
|
242
|
+
}
|
243
|
+
dout = {
|
244
|
+
'a' => 1,
|
245
|
+
'b:c:e' => 100,
|
246
|
+
'b:d' => 200,
|
247
|
+
}
|
248
|
+
expect(Scalyr::Common::Util.flatten(din, ':', true, true)).to eq(dout)
|
249
|
+
end
|
250
|
+
|
215
251
|
it "stringifies non-string keys" do
|
216
252
|
din = {
|
217
253
|
'a' => 1,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.19.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-07-
|
11
|
+
date: 2021-07-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|