logstash-output-scalyr 0.1.19.beta → 0.1.20.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '084d41a09e7ef86214868f26a97639169df8982d9488eb827d337ef53f02fc07'
4
- data.tar.gz: 5bd3b17804c60044901d2f40c40419c119e680f7ac4a4ffc543701a885cb0c2e
3
+ metadata.gz: 23ede2e07ed42483146d4a267c8517fed2f8639fb27998e3f37e8732bd50a45d
4
+ data.tar.gz: 4d3594dd94a908d5852d58fc6e5214982ae731ef8b3de3eb3707455bfa77bd3d
5
5
  SHA512:
6
- metadata.gz: 837bd2d4d142f31bcd30494175731818de9f60f9f39ebc13cf0ae92b36e58ad4b6d288ba2f92109323f096d30a1774334651907b07e1d1310f6b42dd24698b2e
7
- data.tar.gz: 221e47feab585a99dc69b475f446f9a9b4f9d2bd91ffd60753886438c348e0139618afa46f3889180aba68582ed9532376ee5449fb0cf91bb05cbb796c11a08c
6
+ metadata.gz: b11f8aaeea8530b316de904d2fd4b654b8b2a33c2c13ca10614ca3cb346a154e494d9ec7c831a1cf0a83449101ed2bb59c97cc5506870bd252c31cbf9dbc9052
7
+ data.tar.gz: dc6a23b5eae108b7ac8818cfd018c9a9670f69c31fb582b7c8568ebb33720fef19564b54420b2cfc4e7fc04e4a742836bdd2eefc013b15e240379726858fe15f
data/CHANGELOG.md CHANGED
@@ -1,5 +1,10 @@
1
1
  # Beta
2
2
 
3
+ ## 0.1.20.beta
4
+ - Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
5
+ - Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
6
+ It limits the maximum amount of keys we can have in the final flattened record. Defaults to unlimited.
7
+
3
8
  ## 0.1.19.beta
4
9
  - Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
5
10
  by setting the `fix_deep_flattening_delimiters` configuration option to true.
data/README.md CHANGED
@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
10
10
  # Quick start
11
11
 
12
12
  1. Build the gem, run `gem build logstash-output-scalyr.gemspec`
13
- 2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.19.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
13
+ 2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.20.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
14
14
  3. Configure the output plugin (e.g. add it to a pipeline .conf)
15
15
  4. Restart Logstash
16
16
 
@@ -71,6 +71,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
71
71
  config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
72
72
  config :flatten_nested_arrays, :validate => :boolean, :default => true
73
73
  config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
74
+ config :flattening_max_key_count, :validate => :number, :default => -1
74
75
 
75
76
  # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
76
77
  # :flat_tag_value
@@ -633,7 +634,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
633
634
  # flatten record
634
635
  if @flatten_nested_values
635
636
  start_time = Time.now.to_f
636
- record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters)
637
+ begin
638
+ record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
639
+ rescue Scalyr::Common::Util::MaxKeyCountError => e
640
+ @logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
641
+ end
637
642
  end_time = Time.now.to_f
638
643
  flatten_nested_values_duration = end_time - start_time
639
644
  end
@@ -1,52 +1,96 @@
1
1
  module Scalyr; module Common; module Util;
2
2
 
3
+ class MaxKeyCountError < StandardError
4
+ attr_reader :message, :sample_keys
5
+
6
+ def initialize(message, sample_keys)
7
+ @message = message
8
+ @sample_keys = sample_keys
9
+ end
10
+ end
3
11
 
4
12
  # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
5
13
  # nested keys. Returned keys are always strings. If a non-hash or array is provided, raises TypeError.
6
14
  # Please see rspec util_spec.rb for expected behavior.
7
15
  # Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
8
16
  # because some queries and dashboards already rely on the broken functionality.
9
- def self.flatten(obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false)
17
+ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false, max_key_count=-1)
10
18
 
11
19
  # base case is input object is not enumerable, in which case simply return it
12
- if !obj.respond_to?(:each)
20
+ if !hash_obj.respond_to?(:each)
13
21
  raise TypeError.new('Input must be a hash or array')
14
22
  end
23
+ # case where we pass in a valid array, but don't want to flatten arrays
24
+ if !hash_obj.respond_to?(:has_key?) and !flatten_arrays
25
+ return hash_obj
26
+ end
15
27
 
28
+ stack = []
29
+ stack << hash_obj
30
+ key_stack = []
31
+ key_stack << ""
32
+ key_list = []
33
+ key_list_width = []
16
34
  result = Hash.new
17
- # require 'pry'
18
- # binding.pry
19
-
20
- if obj.respond_to?(:has_key?)
21
-
22
- # input object is a hash
23
- obj.each do |key, value|
24
- if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
25
- flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
26
- result["#{key}#{delimiter}#{subkey}"] = subvalue
27
- end
28
- else
29
- result["#{key}"] = value
35
+ test_key = 0
36
+ #Debugging
37
+ #require 'pry'
38
+ #binding.pry
39
+
40
+ until stack.empty?
41
+ obj = stack.pop
42
+ key_list << key_stack.pop
43
+
44
+ # Case when object is a hash
45
+ if obj.respond_to?(:has_key?)
46
+ key_list_width << obj.keys.count
47
+ obj.each do |key, value|
48
+ key_stack << key
49
+ stack << value
30
50
  end
31
- end
32
51
 
33
- elsif flatten_arrays
52
+ # Case when object is an array we intend to flatten
53
+ elsif flatten_arrays and obj.respond_to?(:each)
54
+ key_list_width << obj.count
55
+ obj.each_with_index do |value, index|
56
+ key_stack << index
57
+ stack << value
58
+ end
34
59
 
35
- # input object is an array or set
36
- obj.each_with_index do |value, index|
37
- if value.respond_to?(:each)
38
- flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
39
- result["#{index}#{delimiter}#{subkey}"] = subvalue
60
+ else
61
+ result_key = ""
62
+ delim = delimiter
63
+ key_list.each_with_index do |key, index|
64
+ # We have a blank key at the start of the key list to avoid issues with calling pop, so we ignore delimiter
65
+ # for the first two keys
66
+ if index > 1
67
+ result_key += "#{delim}#{key}"
68
+ if not fix_deep_flattening_delimiters
69
+ delim = "_"
70
+ end
71
+ else
72
+ result_key += "#{key}"
40
73
  end
41
- else
42
- result["#{index}"] = value
43
74
  end
44
- end
75
+ result[result_key] = obj
45
76
 
46
- else
77
+ if max_key_count > -1 and result.keys.count > max_key_count
78
+ raise MaxKeyCountError.new(
79
+ "Resulting flattened object will contain more keys than the configured flattening_max_key_count of #{max_key_count}",
80
+ result.keys[0..6]
81
+ )
82
+ end
47
83
 
48
- result = obj
84
+ throw_away = key_list.pop
85
+ until key_list_width.empty? or key_list_width[-1] > 1
86
+ throw_away = key_list_width.pop
87
+ throw_away = key_list.pop
88
+ end
89
+ if not key_list_width.empty?
90
+ key_list_width[-1] -= 1
91
+ end
49
92
 
93
+ end
50
94
  end
51
95
 
52
96
  return result
@@ -1,2 +1,2 @@
1
1
  # encoding: utf-8
2
- PLUGIN_VERSION = "v0.1.19.beta"
2
+ PLUGIN_VERSION = "v0.1.20.beta"
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-scalyr'
3
- s.version = '0.1.19.beta'
3
+ s.version = '0.1.20.beta'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Scalyr output plugin for Logstash"
6
6
  s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -409,6 +409,38 @@ describe LogStash::Outputs::Scalyr do
409
409
  end
410
410
  end
411
411
 
412
+ context "when configured to flatten with max keys configured to 3" do
413
+ config = {
414
+ 'api_write_token' => '1234',
415
+ 'flatten_nested_values' => true, # this converts into string 'true'
416
+ 'flattening_max_key_count' => 3,
417
+ }
418
+ plugin = LogStash::Outputs::Scalyr.new(config)
419
+ it "does not flatten" do
420
+ allow(plugin).to receive(:send_status).and_return(nil)
421
+ plugin.register
422
+ allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
423
+ result = plugin.build_multi_event_request_array(sample_events)
424
+ body = JSON.parse(result[0][:body])
425
+ expect(body['events'].size).to eq(3)
426
+ expect(body['events'][2]['attrs']).to eq({
427
+ "nested" => {'a'=>1, 'b'=>[3,4,5]},
428
+ 'seq' => 3,
429
+ 'source_file' => 'my file 3',
430
+ 'source_host' => 'my host 3',
431
+ 'serverHost' => 'Logstash',
432
+ "tags" => ["t1", "t2", "t3"],
433
+ "parser" => "logstashParser",
434
+ })
435
+ expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error while flattening record",
436
+ {
437
+ :error_message=>"Resulting flattened object will contain more keys than the configured flattening_max_key_count of 3",
438
+ :sample_keys=>["serverHost", "parser", "tags_2", "tags_1"]
439
+ }
440
+ ).exactly(3).times
441
+ end
442
+ end
443
+
412
444
  context "when receiving an event with Bignums" do
413
445
  config = {
414
446
  'api_write_token' => '1234',
@@ -283,4 +283,23 @@ describe Scalyr::Common::Util do
283
283
  it "raises exception if a non-dict is provided" do
284
284
  expect {Scalyr::Common::Util.flatten(1)}.to raise_error(TypeError)
285
285
  end
286
+
287
+ it "flattens a hash 5000 layers deep" do
288
+ din = {
289
+ 'a' => {},
290
+ }
291
+ hash = din
292
+ for i in 0...4999
293
+ hash = hash["a"]
294
+ hash["a"] = {}
295
+ if i == 4998
296
+ hash["a"] = "b"
297
+ end
298
+ end
299
+
300
+ dout = {
301
+ 'a' + "_a" * 4999 => "b",
302
+ }
303
+ expect(Scalyr::Common::Util.flatten(din, '_')).to eq(dout)
304
+ end
286
305
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-scalyr
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.19.beta
4
+ version: 0.1.20.beta
5
5
  platform: ruby
6
6
  authors:
7
7
  - Edward Chee
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-08 00:00:00.000000000 Z
11
+ date: 2021-08-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement