logstash-output-scalyr 0.1.23.beta → 0.1.24.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e2757da74eb03ea09a88597ed713db01bc050e17c31c771558bdb16a80ca40e
-  data.tar.gz: 1bc8519d1b30aea135d66b8b9ecfb632f688fd62abd58cd3b91cc92c4ad8957f
+  metadata.gz: 312d501ddc98f27b8d2be32bb20cb29439e7ce096d5f12870ef1b1bf095c32ee
+  data.tar.gz: c0f6c4a51f42acef8266ee30480fc00a19e59098d7b957deddcc017bade71695
 SHA512:
-  metadata.gz: 327a3e2ba020f3166b0c435700bdfedf4f96598a352f1f66f45fe9e671a053ee684aa812d686a4d58b5dd9f319e8b0f87e2ea8290d876e504b8416a8010795ed
-  data.tar.gz: 1ba44520daddda5c17da810221063ce815af082645ae3fd12e76b031247ff8a13cb3e7dab4fbf69e84e5a876605290702785cc17639de69435f0b780a74691f9
+  metadata.gz: 2628a0bcfbe9a13dfc5bb9fd93360ef7629fca158cc325184817b4f15492ff5fcb28c03ca992c08dbbf9dc7953d2715b071b5d94f67912c36f66625f7bcf3234
+  data.tar.gz: 1d009fe06794e8f2d15737da874609600e57569fe3d6be0297fb23a1ab99d375f3ac2e9bdbf52b653baa85222bc3d5da001ac8d68a3848c8099ed7eb1510175f

data/lib/logstash/outputs/scalyr.rb

@@ -746,6 +746,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       else
         # If size estimation is disabled we simply append the event and handle splitting later on (if needed)
         append_event = true
+        add_bytes = 0
       end
 
       # if we haven't consumed the current event already
@@ -758,9 +759,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     }
 
-    # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
-    multi_event_request_array << multi_event_request
+    # create a final request with any left over events (and make sure there is at least one event)
+    if scalyr_events.size >= 1
+      multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
+      multi_event_request_array << multi_event_request
+    end
+
     multi_event_request_array
   end
 

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.23.beta"
+PLUGIN_VERSION = "v0.1.24.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.23.beta'
+  s.version         = '0.1.24.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_spec.rb

@@ -392,6 +392,124 @@ describe LogStash::Outputs::Scalyr do
       end
     end
 
+    context "split large batches into multiple scalyr requests" do
+      it "estimate_each_event_size is true explicit (default) batch split into 3 scalyr requests" do
+        config = {
+            'api_write_token' => '1234',
+            'flatten_tags' => true,
+            'flat_tag_value' => 'true',
+            'flat_tag_prefix' => 'tag_prefix_',
+            'flatten_nested_values' => true,  # this converts into string 'true'
+            'max_request_buffer' => 10,
+            'estimate_each_event_size' => true
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        expect(result.size).to eq(3)
+
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(1)
+
+        body = JSON.parse(result[1][:body])
+        expect(body['events'].size).to eq(1)
+
+        body = JSON.parse(result[2][:body])
+        expect(body['events'].size).to eq(1)
+        expect(body['events'][0]['attrs']).to eq({
+                                                     "nested_a" => 1,
+                                                     "nested_b_0" => 3,
+                                                     "nested_b_1" => 4,
+                                                     "nested_b_2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+
+      it "estimate_each_event_size is true implicit (default) batch split into 3 scalyr requests" do
+        config = {
+            'api_write_token' => '1234',
+            'flatten_tags' => true,
+            'flat_tag_value' => 'true',
+            'flat_tag_prefix' => 'tag_prefix_',
+            'flatten_nested_values' => true,  # this converts into string 'true'
+            'max_request_buffer' => 10,
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        expect(result.size).to eq(3)
+
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(1)
+
+        body = JSON.parse(result[1][:body])
+        expect(body['events'].size).to eq(1)
+
+        body = JSON.parse(result[2][:body])
+        expect(body['events'].size).to eq(1)
+        expect(body['events'][0]['attrs']).to eq({
+                                                     "nested_a" => 1,
+                                                     "nested_b_0" => 3,
+                                                     "nested_b_1" => 4,
+                                                     "nested_b_2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+
+      it "estimate_each_event_size is false batch not split into multiple scalyr requests" do
+        config = {
+            'api_write_token' => '1234',
+            'flatten_tags' => true,
+            'flat_tag_value' => 'true',
+            'flat_tag_prefix' => 'tag_prefix_',
+            'flatten_nested_values' => true,  # this converts into string 'true'
+            'max_request_buffer' => 10,
+            'estimate_each_event_size' => false
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        expect(result.size).to eq(1)
+
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested_a" => 1,
+                                                     "nested_b_0" => 3,
+                                                     "nested_b_1" => 4,
+                                                     "nested_b_2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
     context "when not configured to flatten values and tags" do
       config = {
           'api_write_token' => '1234',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.23.beta
+  version: 0.1.24.beta
 platform: ruby
 authors:
 - Edward Chee