fluent-plugin-scalyr 0.8.14 → 0.8.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9f9a9d2d7c0086b3885e3c3962fcd7271b84b79cbd1dd937cfb27f1d24e16d8
-  data.tar.gz: d99dd0b487a8f4920a2abc53b00fb2f75499647ed6cb75bcd0ffa9a0a0dd0871
+  metadata.gz: df57f8a053be1230c4af77591eb3b2cab52dd23ce2733a38046a6d03a1ab24cb
+  data.tar.gz: 20675ccb717aa4c71f216c619070d78dcde0123a43f1866aa43387f84e4a856d
 SHA512:
-  metadata.gz: b6374eb7cb7e4eb1fa3ba3c7342999f176743b07b3293ff3a34f87e127004db50502fb3989117873b68702600fdb6072f4fe39f1f78fef2d280d37a86fa04dc8
-  data.tar.gz: 8964f56c050fa025825ef3bb4c74d7ef593d32eee86930f9bb9847bedd5048a1851767cc1d3d9d67f064dc1b08622b479add48cab6f395519f08e692c2f57154
+  metadata.gz: 7036556c268716b93ca61a27b02a2bfd7d1760b6eaecb15423491113c0ba6e97a0472df6d69776fc41b0bb3266e2554944e8853a788020637ccc781d320b786e
+  data.tar.gz: 767f067ee3f88c0af7abd58049f11afd3305f7c0a8ae6885e2a6cca5f57318b1b111869fc3fb11a2e6f61bcd627abdf9e6a47435fab5733cdee826bd1493a741

data/VERSION

@@ -1 +1 @@
-0.8.14
+0.8.17

data/fluent-plugin-scalyr.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "zlib"
   gem.add_development_dependency "bundler", "~> 1.9"
   gem.add_development_dependency "flexmock", "~> 1.2"
-  gem.add_development_dependency "rake", "~> 0.9"
+  gem.add_development_dependency "rake", "~> 13.0"
   gem.add_development_dependency "rubocop", "~> 0.4"
   gem.add_development_dependency "test-unit", "~> 3.0"
 end

data/lib/fluent/plugin/out_scalyr.rb

@@ -73,6 +73,12 @@ module Scalyr
     end
 
     def configure(conf)
+      @version = if Gem.loaded_specs.key?("fluent-plugin-scalyr")
+                   Gem.loaded_specs["fluent-plugin-scalyr"].version
+                 else
+                   "unknown"
+                 end
+
       if conf.elements("buffer").empty?
         $log.warn "Pre 0.14.0 configuration file detected.  Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
       end
@@ -146,7 +152,7 @@ module Scalyr
       # Generate a session id.  This will be called once for each <match> in fluent.conf that uses scalyr
       @session = SecureRandom.uuid
 
-      $log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
+      $log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session} version=#{@version}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
     end
 
     def format(tag, time, record)
@@ -254,8 +260,8 @@ module Scalyr
 
       post = Net::HTTP::Post.new uri.path
       post.add_field("Content-Type", "application/json")
-
       post.add_field("Content-Encoding", encoding) if @compression_type
+      post.add_field("User-Agent", "fluent-plugin-scalyr;#{@version}")
 
       post.body = body
 
@@ -358,13 +364,37 @@ module Scalyr
         # generate new request if json size of events in the array exceed maximum request buffer size
         append_event = true
         if total_bytes + event_json.bytesize > @max_request_buffer
-          # make sure we always have at least one event
+          # the case where a single event causes us to exceed the @max_request_buffer
           if events.empty?
-            events << event
+            # if we are able to truncate the content (and append an ellipsis)
+            # inside the @message_field we do so here
+            if record.key?(@message_field) &&
+              record[@message_field].is_a?(String) &&
+              record[@message_field].bytesize > event_json.bytesize - @max_request_buffer &&
+              record[@message_field].bytesize >= 3
+
+              @log.warn "Received a record that cannot fit within max_request_buffer "\
+                "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
+                "is #{event_json.bytesize}. The #{@message_field} field will be truncated to fit."
+              max_msg_size = @max_request_buffer - event_json.bytesize - 3
+              truncated_msg = event[:attrs][@message_field][0...max_msg_size] + "..."
+              event[:attrs][@message_field] = truncated_msg
+              events << event
+
+            # otherwise we drop the event and save ourselves hitting a 4XX response from the server
+            else
+              @log.warn "Received a record that cannot fit within max_request_buffer "\
+                "(#{@max_request_buffer}) from #{record['logfile']}, serialized event size "\
+                "is #{event_json.bytesize}. The #{@message_field} field too short to truncate, "\
+                "dropping event."
+            end
             append_event = false
           end
-          request = create_request(events, current_threads)
-          requests << request
+
+          unless events.empty?
+            request = create_request(events, current_threads)
+            requests << request
+          end
 
           total_bytes = 0
           current_threads = {}
@@ -380,8 +410,12 @@ module Scalyr
       }
 
       # create a final request with any left over events
-      request = create_request(events, current_threads)
-      requests << request
+      unless events.empty?
+        request = create_request(events, current_threads)
+        requests << request
+      end
+
+      requests
     end
 
     def create_request(events, current_threads)

data/test/test_events.rb

@@ -357,4 +357,48 @@ class EventsTest < Scalyr::ScalyrOutTest
       d.feed(time, attrs)
     end
   end
+
+  def test_truncated_large_event
+    d = create_driver CONFIG + "max_request_buffer 4000"
+
+    time = event_time("2015-04-01 10:00:00 UTC")
+    attrs = {"log" => "this is a test", "message" => "0123456789" * 500}
+
+    response = flexmock(Net::HTTPResponse, code: "200", body: '{ "status":"success" }')
+    mock = flexmock(d.instance)
+
+    mock.should_receive(:post_request).with(
+      URI,
+      on {|request_body|
+        body = JSON.parse(request_body)
+        events = body["events"]
+        assert(events[0]["attrs"].key?("message"), "'message' field not found in event")
+        assert_equal(
+          "0123456789" * 388 + "012...",
+          events[0]["attrs"]["message"],
+          "'message' field incorrect"
+        )
+        true
+      }
+    ).once.and_return(response)
+
+    d.run(default_tag: "test") do
+      d.feed(time, attrs)
+    end
+  end
+
+  def test_dropped_large_event
+    d = create_driver CONFIG + "max_request_buffer 4000"
+
+    time = event_time("2015-04-01 10:00:00 UTC")
+    attrs = {"message" => "this is a test", "not_message" => "0123456789" * 500}
+
+    mock = flexmock(d.instance)
+
+    mock.should_receive(:post_request).never
+
+    d.run(default_tag: "test") do
+      d.feed(time, attrs)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.8.14
+  version: 0.8.17
 platform: ruby
 authors:
 - Imron Alston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-28 00:00:00.000000000 Z
+date: 2022-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -106,14 +106,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -184,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.11
 signing_key: 
 specification_version: 4
 summary: Scalyr plugin for fluentd