logstash-output-scalyr 0.1.19.beta → 0.1.20.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '084d41a09e7ef86214868f26a97639169df8982d9488eb827d337ef53f02fc07'
-  data.tar.gz: 5bd3b17804c60044901d2f40c40419c119e680f7ac4a4ffc543701a885cb0c2e
+  metadata.gz: 23ede2e07ed42483146d4a267c8517fed2f8639fb27998e3f37e8732bd50a45d
+  data.tar.gz: 4d3594dd94a908d5852d58fc6e5214982ae731ef8b3de3eb3707455bfa77bd3d
 SHA512:
-  metadata.gz: 837bd2d4d142f31bcd30494175731818de9f60f9f39ebc13cf0ae92b36e58ad4b6d288ba2f92109323f096d30a1774334651907b07e1d1310f6b42dd24698b2e
-  data.tar.gz: 221e47feab585a99dc69b475f446f9a9b4f9d2bd91ffd60753886438c348e0139618afa46f3889180aba68582ed9532376ee5449fb0cf91bb05cbb796c11a08c
+  metadata.gz: b11f8aaeea8530b316de904d2fd4b654b8b2a33c2c13ca10614ca3cb346a154e494d9ec7c831a1cf0a83449101ed2bb59c97cc5506870bd252c31cbf9dbc9052
+  data.tar.gz: dc6a23b5eae108b7ac8818cfd018c9a9670f69c31fb582b7c8568ebb33720fef19564b54420b2cfc4e7fc04e4a742836bdd2eefc013b15e240379726858fe15f

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Beta
 
+## 0.1.20.beta
+- Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
+- Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
+It limits the maximum amount of keys we can have in the final flattened record. Defaults to unlimited.
+
 ## 0.1.19.beta
 - Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
 by setting the `fix_deep_flattening_delimiters` configuration option to true.

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.19.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.20.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -71,6 +71,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
   config :flatten_nested_arrays, :validate => :boolean, :default => true
   config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
+  config :flattening_max_key_count, :validate => :number, :default => -1
 
   # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
   # :flat_tag_value
@@ -633,7 +634,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters)
+        begin
+          record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
+        rescue Scalyr::Common::Util::MaxKeyCountError => e
+          @logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
+        end
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end

data/lib/scalyr/common/util.rb

@@ -1,52 +1,96 @@
 module Scalyr; module Common; module Util;
 
+class MaxKeyCountError < StandardError
+  attr_reader :message, :sample_keys
+
+  def initialize(message, sample_keys)
+    @message = message
+    @sample_keys = sample_keys
+  end
+end
 
 # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
 # nested keys.  Returned keys are always strings.  If a non-hash or array is provided, raises TypeError.
 # Please see rspec util_spec.rb for expected behavior.
 # Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
 # because some queries and dashboards already rely on the broken functionality.
-def self.flatten(obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false)
+def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false, max_key_count=-1)
 
   # base case is input object is not enumerable, in which case simply return it
-  if !obj.respond_to?(:each)
+  if !hash_obj.respond_to?(:each)
     raise TypeError.new('Input must be a hash or array')
   end
+  # case where we pass in a valid array, but don't want to flatten arrays
+  if !hash_obj.respond_to?(:has_key?) and !flatten_arrays
+    return hash_obj
+  end
 
+  stack = []
+  stack << hash_obj
+  key_stack = []
+  key_stack << ""
+  key_list = []
+  key_list_width = []
   result = Hash.new
-  # require 'pry'
-  # binding.pry
-
-  if obj.respond_to?(:has_key?)
-
-    # input object is a hash
-    obj.each do |key, value|
-      if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
-        flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
-          result["#{key}#{delimiter}#{subkey}"] = subvalue
-        end
-      else
-        result["#{key}"] = value
+  test_key = 0
+  #Debugging
+  #require 'pry'
+  #binding.pry
+
+  until stack.empty?
+    obj = stack.pop
+    key_list << key_stack.pop
+
+    # Case when object is a hash
+    if obj.respond_to?(:has_key?)
+      key_list_width << obj.keys.count
+      obj.each do |key, value|
+        key_stack << key
+        stack << value
       end
-    end
 
-  elsif flatten_arrays
+    # Case when object is an array we intend to flatten
+    elsif flatten_arrays and obj.respond_to?(:each)
+      key_list_width << obj.count
+      obj.each_with_index do |value, index|
+        key_stack << index
+        stack << value
+      end
 
-    # input object is an array or set
-    obj.each_with_index do |value, index|
-      if value.respond_to?(:each)
-        flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
-          result["#{index}#{delimiter}#{subkey}"] = subvalue
+    else
+      result_key = ""
+      delim = delimiter
+      key_list.each_with_index do |key, index|
+        # We have a blank key at the start of the key list to avoid issues with calling pop, so we ignore delimiter
+        # for the first two keys
+        if index > 1
+          result_key += "#{delim}#{key}"
+          if not fix_deep_flattening_delimiters
+            delim = "_"
+          end
+        else
+          result_key += "#{key}"
         end
-      else
-        result["#{index}"] = value
       end
-    end
+      result[result_key] = obj
 
-  else
+      if max_key_count > -1 and result.keys.count > max_key_count
+        raise MaxKeyCountError.new(
+          "Resulting flattened object will contain more keys than the configured flattening_max_key_count of #{max_key_count}",
+          result.keys[0..6]
+        )
+      end
 
-    result = obj
+      throw_away = key_list.pop
+      until key_list_width.empty? or key_list_width[-1] > 1
+        throw_away = key_list_width.pop
+        throw_away = key_list.pop
+      end
+      if not key_list_width.empty?
+        key_list_width[-1] -= 1
+      end
 
+    end
   end
 
   return result

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.19.beta"
+PLUGIN_VERSION = "v0.1.20.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.19.beta'
+  s.version         = '0.1.20.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_spec.rb

@@ -409,6 +409,38 @@ describe LogStash::Outputs::Scalyr do
       end
     end
 
+    context "when configured to flatten with max keys configured to 3" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flattening_max_key_count' => 3,
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "does not flatten" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested" => {'a'=>1, 'b'=>[3,4,5]},
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tags" => ["t1", "t2", "t3"],
+                                                     "parser" => "logstashParser",
+                                                 })
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error while flattening record",
+          {
+            :error_message=>"Resulting flattened object will contain more keys than the configured flattening_max_key_count of 3",
+            :sample_keys=>["serverHost", "parser", "tags_2", "tags_1"]
+          }
+        ).exactly(3).times
+      end
+    end
+
     context "when receiving an event with Bignums" do
       config = {
           'api_write_token' => '1234',

data/spec/scalyr/common/util_spec.rb

@@ -283,4 +283,23 @@ describe Scalyr::Common::Util do
   it "raises exception if a non-dict is provided" do
     expect {Scalyr::Common::Util.flatten(1)}.to raise_error(TypeError)
   end
+
+  it "flattens a hash 5000 layers deep" do
+    din = {
+        'a' => {},
+    }
+    hash = din
+    for i in 0...4999
+      hash = hash["a"]
+      hash["a"] = {}
+      if i == 4998
+        hash["a"] = "b"
+      end
+    end
+
+    dout = {
+        'a' + "_a" * 4999 => "b",
+    }
+    expect(Scalyr::Common::Util.flatten(din, '_')).to eq(dout)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.19.beta
+  version: 0.1.20.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-08 00:00:00.000000000 Z
+date: 2021-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement