fluent-plugin-scalyr 0.8.14 → 0.8.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9f9a9d2d7c0086b3885e3c3962fcd7271b84b79cbd1dd937cfb27f1d24e16d8
-  data.tar.gz: d99dd0b487a8f4920a2abc53b00fb2f75499647ed6cb75bcd0ffa9a0a0dd0871
+  metadata.gz: b707a6d76dc806731c45f6a96915936e559c49c79e873c3c5ea5dce4ffbaf288
+  data.tar.gz: a328a42722469673e69d9e7a77e84c65cb1f7235a566076a1316216937a54405
 SHA512:
-  metadata.gz: b6374eb7cb7e4eb1fa3ba3c7342999f176743b07b3293ff3a34f87e127004db50502fb3989117873b68702600fdb6072f4fe39f1f78fef2d280d37a86fa04dc8
-  data.tar.gz: 8964f56c050fa025825ef3bb4c74d7ef593d32eee86930f9bb9847bedd5048a1851767cc1d3d9d67f064dc1b08622b479add48cab6f395519f08e692c2f57154
+  metadata.gz: fad3abf458ae0f215f7a29ca3aa4cc167d05811e270c3eace46057ce291307f3b76ef8c13c269b8c43b4564909001e2421dd6ef591188e52c6ed44c22a5ef946
+  data.tar.gz: b8a3268ce7d4d71178ea3356ba47ffd9247cbe3c55d49d436916868076710c4f66719266235d926e314b936bed5544a92de6afdad93a0b33be5d4d0fe0d473cb

data/VERSION

@@ -1 +1 @@
-0.8.14
+0.8.15

data/lib/fluent/plugin/out_scalyr.rb

@@ -358,13 +358,37 @@ module Scalyr
         # generate new request if json size of events in the array exceed maximum request buffer size
         append_event = true
         if total_bytes + event_json.bytesize > @max_request_buffer
-          # make sure we always have at least one event
+          # the case where a single event causes us to exceed the @max_request_buffer
           if events.empty?
-            events << event
+            # if we are able to truncate the content (and append an ellipsis)
+            # inside the @message_field we do so here
+            if record.key?(@message_field) &&
+              record[@message_field].is_a?(String) &&
+              record[@message_field].bytesize > event_json.bytesize - @max_request_buffer &&
+              record[@message_field].bytesize >= 3
+
+              @log.warn "Received a record that cannot fit within max_request_buffer "\
+                "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
+                "is #{event_json.bytesize}. The #{@message_field} field will be truncated to fit."
+              max_msg_size = @max_request_buffer - event_json.bytesize - 3
+              truncated_msg = event[:attrs][@message_field][0...max_msg_size] + "..."
+              event[:attrs][@message_field] = truncated_msg
+              events << event
+
+            # otherwise we drop the event and save ourselves hitting a 4XX response from the server
+            else
+              @log.warn "Received a record that cannot fit within max_request_buffer "\
+                "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
+                "is #{event_json.bytesize}. The #{@message_field} field too short to truncate, "\
+                "dropping event."
+            end
             append_event = false
           end
-          request = create_request(events, current_threads)
-          requests << request
+
+          unless events.empty?
+            request = create_request(events, current_threads)
+            requests << request
+          end
 
           total_bytes = 0
           current_threads = {}
@@ -380,8 +404,12 @@ module Scalyr
       }
 
       # create a final request with any left over events
-      request = create_request(events, current_threads)
-      requests << request
+      unless events.empty?
+        request = create_request(events, current_threads)
+        requests << request
+      end
+
+      requests
     end
 
     def create_request(events, current_threads)

data/test/test_events.rb

@@ -357,4 +357,48 @@ class EventsTest < Scalyr::ScalyrOutTest
       d.feed(time, attrs)
     end
   end
+
+  def test_truncated_large_event
+    d = create_driver CONFIG + "max_request_buffer 4000"
+
+    time = event_time("2015-04-01 10:00:00 UTC")
+    attrs = {"log" => "this is a test", "message" => "0123456789" * 500}
+
+    response = flexmock(Net::HTTPResponse, code: "200", body: '{ "status":"success" }')
+    mock = flexmock(d.instance)
+
+    mock.should_receive(:post_request).with(
+      URI,
+      on {|request_body|
+        body = JSON.parse(request_body)
+        events = body["events"]
+        assert(events[0]["attrs"].key?("message"), "'message' field not found in event")
+        assert_equal(
+          "0123456789" * 388 + "012...",
+          events[0]["attrs"]["message"],
+          "'message' field incorrect"
+        )
+        true
+      }
+    ).once.and_return(response)
+
+    d.run(default_tag: "test") do
+      d.feed(time, attrs)
+    end
+  end
+
+  def test_dropped_large_event
+    d = create_driver CONFIG + "max_request_buffer 4000"
+
+    time = event_time("2015-04-01 10:00:00 UTC")
+    attrs = {"message" => "this is a test", "not_message" => "0123456789" * 500}
+
+    mock = flexmock(d.instance)
+
+    mock.should_receive(:post_request).never
+
+    d.run(default_tag: "test") do
+      d.feed(time, attrs)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.8.14
+  version: 0.8.15
 platform: ruby
 authors:
 - Imron Alston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-28 00:00:00.000000000 Z
+date: 2022-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi