logstash-output-scalyr 0.1.19.beta → 0.1.20.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +1 -1
- data/lib/logstash/outputs/scalyr.rb +6 -1
- data/lib/scalyr/common/util.rb +71 -27
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/logstash/outputs/scalyr_spec.rb +32 -0
- data/spec/scalyr/common/util_spec.rb +19 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 23ede2e07ed42483146d4a267c8517fed2f8639fb27998e3f37e8732bd50a45d
|
4
|
+
data.tar.gz: 4d3594dd94a908d5852d58fc6e5214982ae731ef8b3de3eb3707455bfa77bd3d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b11f8aaeea8530b316de904d2fd4b654b8b2a33c2c13ca10614ca3cb346a154e494d9ec7c831a1cf0a83449101ed2bb59c97cc5506870bd252c31cbf9dbc9052
|
7
|
+
data.tar.gz: dc6a23b5eae108b7ac8818cfd018c9a9670f69c31fb582b7c8568ebb33720fef19564b54420b2cfc4e7fc04e4a742836bdd2eefc013b15e240379726858fe15f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,10 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.1.20.beta
|
4
|
+
- Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
|
5
|
+
- Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
|
6
|
+
It limits the maximum amount of keys we can have in the final flattened record. Defaults to unlimited.
|
7
|
+
|
3
8
|
## 0.1.19.beta
|
4
9
|
- Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
|
5
10
|
by setting the `fix_deep_flattening_delimiters` configuration option to true.
|
data/README.md
CHANGED
@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
|
|
10
10
|
# Quick start
|
11
11
|
|
12
12
|
1. Build the gem, run `gem build logstash-output-scalyr.gemspec`
|
13
|
-
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.
|
13
|
+
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.20.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
|
14
14
|
3. Configure the output plugin (e.g. add it to a pipeline .conf)
|
15
15
|
4. Restart Logstash
|
16
16
|
|
@@ -71,6 +71,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
71
71
|
config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
|
72
72
|
config :flatten_nested_arrays, :validate => :boolean, :default => true
|
73
73
|
config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
|
74
|
+
config :flattening_max_key_count, :validate => :number, :default => -1
|
74
75
|
|
75
76
|
# If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
|
76
77
|
# :flat_tag_value
|
@@ -633,7 +634,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
633
634
|
# flatten record
|
634
635
|
if @flatten_nested_values
|
635
636
|
start_time = Time.now.to_f
|
636
|
-
|
637
|
+
begin
|
638
|
+
record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
|
639
|
+
rescue Scalyr::Common::Util::MaxKeyCountError => e
|
640
|
+
@logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
|
641
|
+
end
|
637
642
|
end_time = Time.now.to_f
|
638
643
|
flatten_nested_values_duration = end_time - start_time
|
639
644
|
end
|
data/lib/scalyr/common/util.rb
CHANGED
@@ -1,52 +1,96 @@
|
|
1
1
|
module Scalyr; module Common; module Util;
|
2
2
|
|
3
|
+
class MaxKeyCountError < StandardError
|
4
|
+
attr_reader :message, :sample_keys
|
5
|
+
|
6
|
+
def initialize(message, sample_keys)
|
7
|
+
@message = message
|
8
|
+
@sample_keys = sample_keys
|
9
|
+
end
|
10
|
+
end
|
3
11
|
|
4
12
|
# Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
|
5
13
|
# nested keys. Returned keys are always strings. If a non-hash or array is provided, raises TypeError.
|
6
14
|
# Please see rspec util_spec.rb for expected behavior.
|
7
15
|
# Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
|
8
16
|
# because some queries and dashboards already rely on the broken functionality.
|
9
|
-
def self.flatten(
|
17
|
+
def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false, max_key_count=-1)
|
10
18
|
|
11
19
|
# base case is input object is not enumerable, in which case simply return it
|
12
|
-
if !
|
20
|
+
if !hash_obj.respond_to?(:each)
|
13
21
|
raise TypeError.new('Input must be a hash or array')
|
14
22
|
end
|
23
|
+
# case where we pass in a valid array, but don't want to flatten arrays
|
24
|
+
if !hash_obj.respond_to?(:has_key?) and !flatten_arrays
|
25
|
+
return hash_obj
|
26
|
+
end
|
15
27
|
|
28
|
+
stack = []
|
29
|
+
stack << hash_obj
|
30
|
+
key_stack = []
|
31
|
+
key_stack << ""
|
32
|
+
key_list = []
|
33
|
+
key_list_width = []
|
16
34
|
result = Hash.new
|
17
|
-
|
18
|
-
#
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
obj
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
35
|
+
test_key = 0
|
36
|
+
#Debugging
|
37
|
+
#require 'pry'
|
38
|
+
#binding.pry
|
39
|
+
|
40
|
+
until stack.empty?
|
41
|
+
obj = stack.pop
|
42
|
+
key_list << key_stack.pop
|
43
|
+
|
44
|
+
# Case when object is a hash
|
45
|
+
if obj.respond_to?(:has_key?)
|
46
|
+
key_list_width << obj.keys.count
|
47
|
+
obj.each do |key, value|
|
48
|
+
key_stack << key
|
49
|
+
stack << value
|
30
50
|
end
|
31
|
-
end
|
32
51
|
|
33
|
-
|
52
|
+
# Case when object is an array we intend to flatten
|
53
|
+
elsif flatten_arrays and obj.respond_to?(:each)
|
54
|
+
key_list_width << obj.count
|
55
|
+
obj.each_with_index do |value, index|
|
56
|
+
key_stack << index
|
57
|
+
stack << value
|
58
|
+
end
|
34
59
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
60
|
+
else
|
61
|
+
result_key = ""
|
62
|
+
delim = delimiter
|
63
|
+
key_list.each_with_index do |key, index|
|
64
|
+
# We have a blank key at the start of the key list to avoid issues with calling pop, so we ignore delimiter
|
65
|
+
# for the first two keys
|
66
|
+
if index > 1
|
67
|
+
result_key += "#{delim}#{key}"
|
68
|
+
if not fix_deep_flattening_delimiters
|
69
|
+
delim = "_"
|
70
|
+
end
|
71
|
+
else
|
72
|
+
result_key += "#{key}"
|
40
73
|
end
|
41
|
-
else
|
42
|
-
result["#{index}"] = value
|
43
74
|
end
|
44
|
-
|
75
|
+
result[result_key] = obj
|
45
76
|
|
46
|
-
|
77
|
+
if max_key_count > -1 and result.keys.count > max_key_count
|
78
|
+
raise MaxKeyCountError.new(
|
79
|
+
"Resulting flattened object will contain more keys than the configured flattening_max_key_count of #{max_key_count}",
|
80
|
+
result.keys[0..6]
|
81
|
+
)
|
82
|
+
end
|
47
83
|
|
48
|
-
|
84
|
+
throw_away = key_list.pop
|
85
|
+
until key_list_width.empty? or key_list_width[-1] > 1
|
86
|
+
throw_away = key_list_width.pop
|
87
|
+
throw_away = key_list.pop
|
88
|
+
end
|
89
|
+
if not key_list_width.empty?
|
90
|
+
key_list_width[-1] -= 1
|
91
|
+
end
|
49
92
|
|
93
|
+
end
|
50
94
|
end
|
51
95
|
|
52
96
|
return result
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
PLUGIN_VERSION = "v0.1.
|
2
|
+
PLUGIN_VERSION = "v0.1.20.beta"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.20.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -409,6 +409,38 @@ describe LogStash::Outputs::Scalyr do
|
|
409
409
|
end
|
410
410
|
end
|
411
411
|
|
412
|
+
context "when configured to flatten with max keys configured to 3" do
|
413
|
+
config = {
|
414
|
+
'api_write_token' => '1234',
|
415
|
+
'flatten_nested_values' => true, # this converts into string 'true'
|
416
|
+
'flattening_max_key_count' => 3,
|
417
|
+
}
|
418
|
+
plugin = LogStash::Outputs::Scalyr.new(config)
|
419
|
+
it "does not flatten" do
|
420
|
+
allow(plugin).to receive(:send_status).and_return(nil)
|
421
|
+
plugin.register
|
422
|
+
allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
|
423
|
+
result = plugin.build_multi_event_request_array(sample_events)
|
424
|
+
body = JSON.parse(result[0][:body])
|
425
|
+
expect(body['events'].size).to eq(3)
|
426
|
+
expect(body['events'][2]['attrs']).to eq({
|
427
|
+
"nested" => {'a'=>1, 'b'=>[3,4,5]},
|
428
|
+
'seq' => 3,
|
429
|
+
'source_file' => 'my file 3',
|
430
|
+
'source_host' => 'my host 3',
|
431
|
+
'serverHost' => 'Logstash',
|
432
|
+
"tags" => ["t1", "t2", "t3"],
|
433
|
+
"parser" => "logstashParser",
|
434
|
+
})
|
435
|
+
expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error while flattening record",
|
436
|
+
{
|
437
|
+
:error_message=>"Resulting flattened object will contain more keys than the configured flattening_max_key_count of 3",
|
438
|
+
:sample_keys=>["serverHost", "parser", "tags_2", "tags_1"]
|
439
|
+
}
|
440
|
+
).exactly(3).times
|
441
|
+
end
|
442
|
+
end
|
443
|
+
|
412
444
|
context "when receiving an event with Bignums" do
|
413
445
|
config = {
|
414
446
|
'api_write_token' => '1234',
|
@@ -283,4 +283,23 @@ describe Scalyr::Common::Util do
|
|
283
283
|
it "raises exception if a non-dict is provided" do
|
284
284
|
expect {Scalyr::Common::Util.flatten(1)}.to raise_error(TypeError)
|
285
285
|
end
|
286
|
+
|
287
|
+
it "flattens a hash 5000 layers deep" do
|
288
|
+
din = {
|
289
|
+
'a' => {},
|
290
|
+
}
|
291
|
+
hash = din
|
292
|
+
for i in 0...4999
|
293
|
+
hash = hash["a"]
|
294
|
+
hash["a"] = {}
|
295
|
+
if i == 4998
|
296
|
+
hash["a"] = "b"
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
dout = {
|
301
|
+
'a' + "_a" * 4999 => "b",
|
302
|
+
}
|
303
|
+
expect(Scalyr::Common::Util.flatten(din, '_')).to eq(dout)
|
304
|
+
end
|
286
305
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-08-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|