fluent-plugin-scalyr 0.8.11 → 0.8.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d9fa05bade6ec51a39896e90bce7ddfdf233c8888d5ca2ac41393966c8a924b
-  data.tar.gz: b6406c122b1180faa48294b47ed2ebb1b61aad08c64c0310385a00c0ae8a044a
+  metadata.gz: 1e738c523b1cea2d7aa0976064cb4fa85379b63795f1230a965da934033bc619
+  data.tar.gz: '09241b0eabc17074c5b4effb5f53ba47d183bb97ad724a064d6e9cfa9d5e8641'
 SHA512:
-  metadata.gz: 3b083816314fdf992eaca0cb0216795e2c7a6c1cc7e2f32669452201c45b29d6254dcbc2f69a665d6a0852242423bd87cdf62a88feda5f463e6f774b142d0520
-  data.tar.gz: 8a94b4b649cfd45fa1143e6866d119811236771980f508d0643b0b51b1d6bbdb2e241233a469086a0b5a3e22529c08d1c23b7825966fe78e367defd4672d20c2
+  metadata.gz: 238b972ec9340a013d473911c05955e0a39d03f706d215167f2b3837bf46610917d2741680fd698c5d1075d1f7a307debd3c7cf4111ebba07280110bff2239dc
+  data.tar.gz: 1c42dfccbfa8963652198b3b2c0da102feda2b1052370b3296c26c781f5ab1fcd0126b513f87139e055c437c82347c597fe930f3739b6f61a4f8ac7d1764996a

data/Rakefile

@@ -9,6 +9,7 @@ Rake::TestTask.new(:test) do |test|
   test.libs << "lib" << "test"
   test.test_files = FileList["test/test_*.rb"]
   test.verbose = true
+  test.options = "--verbose=verbose"
 end
 
 task default: [:build]

data/VERSION

@@ -1 +1 @@
-0.8.11
+0.8.12

data/lib/fluent/plugin/out_scalyr.rb

@@ -19,6 +19,7 @@
 
 require "fluent/plugin/output"
 require "fluent/plugin/scalyr_exceptions"
+require "fluent/plugin/scalyr_utils"
 require "fluent/plugin_helper/compat_parameters"
 require "json"
 require "net/http"
@@ -331,16 +332,22 @@ module Scalyr
         begin
           event_json = event.to_json
         rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
-          $log.warn "#{e.class}: #{e.message}"
+          $log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
 
           # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
           time = Fluent::EventTime.new(sec, nsec)
           router.emit_error_event(tag, time, record, e)
 
+          # Print attribute values for debugging / troubleshooting purposes
+          $log.debug "Event attributes:"
+
           event[:attrs].each do |key, value|
-            $log.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
-            event[:attrs][key] = value.encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
+            # NOTE: value doesn't always value.encoding attribute so we use .class which is always available
+            $log.debug "\t#{key} (#{value.class}): '#{value}'"
           end
+
+          # Recursively re-encode and sanitize potentially bad string values
+          event[:attrs] = sanitize_and_reencode_value(event[:attrs])
           event_json = event.to_json
         end
 

data/lib/fluent/plugin/scalyr_utils.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+#
+# Scalyr Output Plugin for Fluentd
+#
+# Copyright (C) 2020 Scalyr, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Scalyr
+  def sanitize_and_reencode_value(value)
+    # Method which recursively sanitizes the provided value and tries to re-encode all the strings as
+    # UTF-8 ignoring any bad unicode sequences
+    case value
+    when Hash
+      return sanitize_and_reencode_hash(value)
+    when Array
+      return sanitize_and_reencode_array(value)
+    when String
+      value = sanitize_and_reencode_string(value)
+      return value
+    end
+
+    # We only need to re-encode strings, for other value types (ints, nils,
+    # etc. no reencoding is needed)
+    value
+  end
+
+  def sanitize_and_reencode_array(array)
+    array.each_with_index do |value, index|
+      value = sanitize_and_reencode_value(value)
+      array[index] = value
+    end
+
+    array
+  end
+
+  def sanitize_and_reencode_hash(hash)
+    hash.each do |key, value|
+      hash[key] = sanitize_and_reencode_value(value)
+    end
+
+    hash
+  end
+
+  def sanitize_and_reencode_string(value)
+    # Function which sanitized the provided string value and tries to re-encode it as UTF-8
+    # ignoring any encoding error which could arise due to bad or partial unicode sequence
+    begin # rubocop:disable Style/RedundantBegin
+      value.encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
+    rescue # rubocop:disable Style/RescueStandardError
+      "failed-to-reencode-as-utf8"
+    end
+  end
+end

data/test/test_events.rb

@@ -170,6 +170,88 @@ class EventsTest < Scalyr::ScalyrOutTest
     assert_equal(mock_called, true, "mock method was never called!")
   end
 
+  def test_build_add_events_body_non_json_serializable_value
+    d = create_driver
+
+    time = event_time("2015-04-01 10:00:00 UTC")
+    attrs = {"a" => 1}
+    attrs["int1"] = 1_601_923_119
+    attrs["int2"] = Integer(1_601_923_119)
+    attrs["int3"] = Integer(9_223_372_036_854_775_807)
+    attrs["int4"] = Integer(-1)
+    attrs["nil"] = nil
+    attrs["array"] = [1, 2, "a", "b", nil]
+    attrs["hash"] = {
+      "a" => "1",
+      "b" => "c"
+    }
+    attrs["logfile"] = "/some/log/file"
+
+    # This partial unicode sequence will fail encoding so we make sure it doesn't break the plugin
+    # and we correctly cast it to a value which we can send to the API
+    attrs["partial_unicode_sequence"] = "\xC2"
+    attrs["array_with_partial_unicode_sequence"] = [1, 2, "a", "b", nil, "7", "\xC2"]
+    attrs["nested_array_with_partial_unicode_sequence"] = [1, 2, "a", "b", nil, "7",
+                                                           [8, 9, [10, "\xC2"]],
+                                                           {"a" => 1, "b" => "\xC2"}]
+    attrs["hash_with_partial_unicode_sequence"] = {
+      "a" => "1",
+      "b" => "\xC2",
+      "c" => nil
+    }
+    attrs["nested_hash_with_partial_unicode_sequence"] = {
+      "a" => "1",
+      "b" => {
+        "c" => "\xC2",
+        "d" => "e",
+        "f" => nil,
+        "g" => {
+          "h" => "\xC2",
+          "b" => 3
+        }
+      }
+    }
+
+    response = flexmock(Net::HTTPResponse, code: "200", body: '{ "status":"success" }')
+    mock = flexmock(d.instance)
+
+    mock_called = false
+
+    # NOTE: We need to perform a deep clone / copy
+    expected_attrs = Marshal.load(Marshal.dump(attrs))
+
+    expected_attrs["partial_unicode_sequence"] = "<?>"
+    expected_attrs["array_with_partial_unicode_sequence"][-1] = "<?>"
+    expected_attrs["nested_array_with_partial_unicode_sequence"][-2][-1][-1] = "<?>"
+    expected_attrs["nested_array_with_partial_unicode_sequence"][-1]["b"] = "<?>"
+    expected_attrs["hash_with_partial_unicode_sequence"]["b"] = "<?>"
+    expected_attrs["nested_hash_with_partial_unicode_sequence"]["b"]["c"] = "<?>"
+    expected_attrs["nested_hash_with_partial_unicode_sequence"]["b"]["g"]["h"] = "<?>"
+
+    # Verify that clone / copy was correct and the original object wasn't modified
+    assert_not_equal(expected_attrs, attrs, "Objects are the same but should be different")
+    assert_not_equal(Marshal.load(Marshal.dump(attrs)), Marshal.load(Marshal.dump(expected_attrs)))
+    assert_equal(attrs["partial_unicode_sequence"], "\xC2")
+    assert_equal(attrs["array_with_partial_unicode_sequence"][-1], "\xC2")
+    assert_equal(attrs["nested_hash_with_partial_unicode_sequence"]["b"]["g"]["h"], "\xC2")
+
+    mock.should_receive(:post_request).with(
+      URI,
+      on {|request_body|
+        body = JSON.parse(request_body)
+        assert_equal(expected_attrs, body["events"][0]["attrs"], "Value of attrs differs from log")
+        mock_called = true
+        true
+      }
+    ).once.and_return(response)
+
+    d.run(default_tag: "test") do
+      d.feed(time, attrs)
+    end
+
+    assert_equal(mock_called, true, "mock method was never called!")
+  end
+
   def test_default_message_field
     d = create_driver CONFIG
 

data/test/test_utils.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+#
+# Scalyr Output Plugin for Fluentd
+#
+# Copyright (C) 2015 Scalyr, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "helper"
+require "flexmock/test_unit"
+
+require "fluent/plugin/scalyr_utils"
+include Scalyr # rubocop:disable Style/MixinUsage
+
+module Scalyr
+  class UtilsTest < Test::Unit::TestCase
+    def test_sanitize_and_reencode_value_simple_types
+      # Simple value - string
+      result = sanitize_and_reencode_value("test foo")
+      assert_equal("test foo", result)
+
+      # Simple value - string with bad unicode sequences
+      result = sanitize_and_reencode_value("test \xC2 foo \xC2 bar")
+      assert_equal("test <?> foo <?> bar", result)
+
+      # Simple value - int
+      result = sanitize_and_reencode_value(100)
+      assert_equal(100, result)
+
+      # Simple value - nill
+      result = sanitize_and_reencode_value(nil)
+      assert_equal(nil, result)
+
+      # Simple value - bool
+      result = sanitize_and_reencode_value(true)
+      assert_equal(true, result)
+
+      result = sanitize_and_reencode_value(false)
+      assert_equal(false, result)
+    end
+
+    def test_sanitize_and_reencode_value_complex_nested_types
+      actual = [1, 2, "a", "b", nil, "7", "\xC2"]
+      expected = [1, 2, "a", "b", nil, "7", "<?>"]
+
+      result = sanitize_and_reencode_value(actual)
+      assert_equal(expected, result)
+
+      actual = [1, 2, "a", "b", nil, "7",
+                [8, 9, [10, "\xC2"]],
+                {"a" => 1, "b" => "\xC2"}]
+      expected = [1, 2, "a", "b", nil, "7",
+                  [8, 9, [10, "<?>"]],
+                  {"a" => 1, "b" => "<?>"}]
+
+      result = sanitize_and_reencode_value(actual)
+      assert_equal(expected, result)
+
+      actual = {
+        "a" => "1",
+        "b" => {
+          "c" => "\xC2",
+          "d" => "e",
+          "f" => nil,
+          "g" => {
+            "h" => "bar \xC2",
+            "b" => 3,
+            "l" => [1, 2, "foo\xC2", 3, 4, 5]
+          }
+        }
+      }
+      expected = {
+        "a" => "1",
+        "b" => {
+          "c" => "<?>",
+          "d" => "e",
+          "f" => nil,
+          "g" => {
+            "h" => "bar <?>",
+            "b" => 3,
+            "l" => [1, 2, "foo<?>", 3, 4, 5]
+          }
+        }
+      }
+      result = sanitize_and_reencode_value(actual)
+      assert_equal(expected, result)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.8.11
+  version: 0.8.12
 platform: ruby
 authors:
 - Imron Alston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-19 00:00:00.000000000 Z
+date: 2020-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -158,11 +158,13 @@ files:
 - fluent.conf.sample
 - lib/fluent/plugin/out_scalyr.rb
 - lib/fluent/plugin/scalyr_exceptions.rb
+- lib/fluent/plugin/scalyr_utils.rb
 - test/helper.rb
 - test/test_config.rb
 - test/test_events.rb
 - test/test_handle_response.rb
 - test/test_ssl_verify.rb
+- test/test_utils.rb
 homepage: https://github.com/scalyr/scalyr-fluentd
 licenses:
 - Apache-2.0
@@ -191,4 +193,5 @@ test_files:
 - test/test_events.rb
 - test/test_handle_response.rb
 - test/test_config.rb
+- test/test_utils.rb
 - test/test_ssl_verify.rb