logstash-output-scalyr 0.1.17.beta → 0.1.18.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96fd1b3adba21150a05f43d1e8d6510711197da866d3cd6ec8b57bae14e53391
-  data.tar.gz: 221aac42711955c49c878c324e2570ebb55050cad13f2d4911b99c4b3f1eb5a1
+  metadata.gz: 9099fcf20201bf8e3905273393a38bb1addefb219e1874a45a439bbfd2dcddbd
+  data.tar.gz: a031faa8657951b66a09c172f6be69b774a2192e661ef99370813abe62250cbb
 SHA512:
-  metadata.gz: c8a678e7caf8778dbe9b44e8d8e48c809b016ac6b89424c4c29c29b63578fea61f236ae80a528dab7707c8ee534cfab0f20a252d688f44df53faccdd1ce907b0
-  data.tar.gz: c263ae0ecffacb454f98f112d384c544cfd68fb0e0e6a0d7339a6b84add2c94b4bb99afd20a815b587f5722e0125d80fc25f9285574187c9f2a0ff2af7f219ec
+  metadata.gz: c25cad81473d0224fb5c75558e302f6383d69361d3263aeaf0fe619b8c2a3a6866bf0eaf811588b9235edb4e15e11a2d0b95ecb99e2a3045ee6c6e66998e1627
+  data.tar.gz: fed7e02e32c70ce199dbd7ec5841711ff77c65d3a56ebe53eeada8c165d5b8fa603fc15b3c00e0e397682a50899dd141bfdcacdcbb97244e2b26205c92802238

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Beta
 
+## 0.1.18.beta
+- Add metrics for successfully sent and failed logstash events, and retries.
+- Make array flattening optional during nested value flattening with the `flatten_nested_arrays` configuration option.
+
 ## 0.1.17.beta
 - Catch errors relating to Bignum conversions present in the ``json`` library and manually convert to string as
 a workaround.

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.17.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.18.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -68,7 +68,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # If true, nested values will be flattened (which changes keys to delimiter-separated concatenation of all
   # nested keys).
   config :flatten_nested_values, :validate => :boolean, :default => false
-  config :flatten_nested_values_delimiter, :validate => :string, :default => "_" 
+  config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
+  config :flatten_nested_arrays, :validate => :boolean, :default => true
 
   # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
   # :flat_tag_value
@@ -241,6 +242,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # level metrics are handled by the HTTP Client class.
     @multi_receive_statistics = {
       :total_multi_receive_secs => 0,
+      :total_events_processed => 0,
+      :successful_events_processed => 0,
+      :failed_events_processed => 0,
+      :total_retry_count => 0,
       :total_java_class_cast_errors => 0
     }
     @plugin_metrics = get_new_metrics
@@ -345,6 +350,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           sleep_interval = sleep_for(sleep_interval)
           exc_sleep += sleep_interval
           exc_retries += 1
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
           message = "Error uploading to Scalyr (will backoff-retry)"
           exc_data = {
               :error_class => e.e_class,
@@ -394,11 +402,19 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           }
           exc_sleep += sleep_interval
           exc_retries += 1
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
           retry if @running and exc_retries < @max_retries
           log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
           next
         end
 
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+          @multi_receive_statistics[:successful_events_processed] += multi_event_request[:logstash_events].length
+        end
+
         if !exc_data.nil?
           message = "Retry successful after error."
           if exc_commonly_retried
@@ -437,6 +453,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
 
   def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+    @stats_lock.synchronize do
+      @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+      @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
+    end
     message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
     sample_events = Array.new
     multi_event_request[:logstash_events][0,5].each {|l_event|
@@ -612,7 +632,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter)
+        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays)
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end

data/lib/scalyr/common/util.rb

@@ -4,7 +4,7 @@ module Scalyr; module Common; module Util;
 # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
 # nested keys.  Returned keys are always strings.  If a non-hash or array is provided, raises TypeError.
 # Please see rspec util_spec.rb for expected behavior.
-def self.flatten(obj, delimiter='_')
+def self.flatten(obj, delimiter='_', flatten_arrays=true)
 
   # base case is input object is not enumerable, in which case simply return it
   if !obj.respond_to?(:each)
@@ -19,8 +19,8 @@ def self.flatten(obj, delimiter='_')
 
     # input object is a hash
     obj.each do |key, value|
-      if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
+      if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
+        flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
           result["#{key}#{delimiter}#{subkey}"] = subvalue
         end
       else
@@ -28,18 +28,23 @@ def self.flatten(obj, delimiter='_')
       end
     end
 
-  else
+  elsif flatten_arrays
 
     # input object is an array or set
     obj.each_with_index do |value, index|
       if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
+        flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
           result["#{index}#{delimiter}#{subkey}"] = subvalue
         end
       else
         result["#{index}"] = value
       end
     end
+
+  else
+
+    result = obj
+
   end
 
   return result

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.17.beta"
+PLUGIN_VERSION = "v0.1.18.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.17.beta'
+  s.version         = '0.1.18.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_spec.rb

@@ -273,9 +273,41 @@ describe LogStash::Outputs::Scalyr do
         expect(body['events'].size).to eq(3)
         expect(body['events'][2]['attrs']).to eq({
                                                      "nested.a" => 1,
-                                                     "nested.b_0" => 3,
-                                                     "nested.b_1" => 4,
-                                                     "nested.b_2" => 5,
+                                                     "nested.b.0" => 3,
+                                                     "nested.b.1" => 4,
+                                                     "nested.b.2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
+    context "when configured to flatten values with custom delimiter, no array flattening" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_arrays' => false,
+          'flatten_nested_values_delimiter' => ".",
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b" => [3, 4, 5],
                                                      'seq' => 3,
                                                      'source_file' => 'my file 3',
                                                      'source_host' => 'my host 3',

data/spec/scalyr/common/util_spec.rb

@@ -132,6 +132,70 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din)).to eq(dout)
   end
 
+  it "flattens a single-level array, no array flattening" do
+    din = [1, 2, 3]
+    dout = [1, 2, 3]
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a multi-level array, no array flattening" do
+    din = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    dout = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
   it "accepts custom delimiters" do
     din = {
         'a' => 1,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.17.beta
+  version: 0.1.18.beta
 platform: ruby
 authors:
 - Edward Chee