fluent-plugin-scalyr 0.8.7 → 0.8.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07de29b14ab7bb4183d662138f847ee9b75e94ee1b4eabeb65bb30725baea31a
-  data.tar.gz: 26261270771892d09baa36c4c7625492b69748c2f908dc9d1595fad609adae6a
+  metadata.gz: 1e738c523b1cea2d7aa0976064cb4fa85379b63795f1230a965da934033bc619
+  data.tar.gz: '09241b0eabc17074c5b4effb5f53ba47d183bb97ad724a064d6e9cfa9d5e8641'
 SHA512:
-  metadata.gz: b2de1513bda2040d60c4f7ccfeb9a44f1c5a8965601833b7f656851d3fe7107e8070b42b6e0f7dca47a31827285670b01317020cea272f8186f56eaeaa97e1e7
-  data.tar.gz: c07a6f90ffda567a2518a5d7de0b7c0af9d0320535ff3b080a1a23d40861a01a5a8193d3c44c03ea97ba0ddb2361f39d012f4dffb766e8e0c9d7be78ddfe703d
+  metadata.gz: 238b972ec9340a013d473911c05955e0a39d03f706d215167f2b3837bf46610917d2741680fd698c5d1075d1f7a307debd3c7cf4111ebba07280110bff2239dc
+  data.tar.gz: 1c42dfccbfa8963652198b3b2c0da102feda2b1052370b3296c26c781f5ab1fcd0126b513f87139e055c437c82347c597fe930f3739b6f61a4f8ac7d1764996a

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gemspec

data/README.md

@@ -1,5 +1,5 @@
 Scalyr output plugin for Fluentd
-=========================
+================================
 
 **Note:** Fluentd introduced breaking changes to their plugin API between
 version 0.12 and 0.14.
@@ -24,7 +24,7 @@ Fluentd may format log messages into json or some other format.  If you want to
   format none
 ```
 
-The Scalyr output plugin assigns a unique Scalyr session id for each Fluentd <match> block.  It is recommended that a single machine doesn't create too many simultaneous Scalyr sessions, so if possible you should try to have a single match for all logs you wish to send to Scalyr.
+The Scalyr output plugin assigns a unique Scalyr session id for each Fluentd <match> block, or for each worker.  It is recommended that a single machine doesn't create too many simultaneous Scalyr sessions, so if possible you should try to have a single match for all logs you wish to send to Scalyr.
 
 This can be done by specifying tags such as scalyr.apache, scalyr.maillog etc and matching on scalyr.\*
 
@@ -33,7 +33,7 @@ Fluentd tag names will be used for the logfile name in Scalyr.
 Scalyr Parsers and Custom Fields
 --------------------------------
 
-You may also need to specify a Scalyr parser for your log message or add custom fields to each log event. This can be done using Fluentd's filter mechanism, in particular the [record_transformer filter](http://docs.fluentd.org/articles/filter_record_transformer).
+You may also need to specify a Scalyr parser for your log message or add custom fields to each log event. This can be done using Fluentd's filter mechanism, in particular the [record_transformer filter](https://docs.fluentd.org/filter/record_transformer).
 
 For example, if you want to use Scalyr's ```accessLog``` parser for all events with the ```scalyr.access``` tag you would add the following to your fluent.conf file:
 
@@ -66,7 +66,9 @@ The following configuration options are also supported:
 
   #scalyr specific options
   api_write_token YOUR_SCALYR_WRITE_TOKEN
-  compression_type bz2
+  compression_type deflate
+  compression_level 6
+  use_hostname_for_serverhost true
   server_attributes {
     "serverHost": "front-1",
     "serverType": "frontend",
@@ -79,7 +81,7 @@ The following configuration options are also supported:
   ssl_verify_depth 5
   message_field message
 
-  max_request_buffer 1048576
+  max_request_buffer 5500000
 
   force_message_encoding nil
   replace_invalid_utf8 false
@@ -91,14 +93,18 @@ The following configuration options are also supported:
     retry_max_interval 30s
     flush_interval 5s
     flush_thread_count 1
-    chunk_limit_size 100k
+    chunk_limit_size 2.5m
     queue_limit_length 1024
   </buffer>
 
 </match>
 ```
 
-####Scalyr specific options
+For some additional examples of configuration for different setups, please refer to the
+[examples/configs/](https://github.com/scalyr/scalyr-fluentd/tree/master/examples/configs/)
+directory.
+
+### Scalyr specific options
 
 ***compression_type*** - compress Scalyr traffic to reduce network traffic. Options are `bz2` and `deflate`. See [here](https://www.scalyr.com/help/scalyr-agent#compressing) for more details.  This feature is optional.
 
@@ -106,9 +112,11 @@ The following configuration options are also supported:
 
 ***server_attributes*** - a JSON hash containing custom server attributes you want to include with each log request.  This value is optional and defaults to *nil*.
 
+***use_hostname_for_serverhost*** - if `true` then if `server_attributes` is nil or it does *not* include a field called `serverHost` then the plugin will add the `serverHost` field with the value set to the hostname that fluentd is running on.  Defaults to `true`.
+
 ***scalyr_server*** - the Scalyr server to send API requests to. This value is optional and defaults to https://agent.scalyr.com/
 
-***ssl_ca_bundle_path*** - a path on your server pointing to a valid certificate bundle.  This value is optional and defaults to */etc/ssl/certs/ca-bundle.crt*.
+***ssl_ca_bundle_path*** - a path on your server pointing to a valid certificate bundle.  This value is optional and defaults to *nil*, which means it will look for a valid certificate bundle on its own.
 
 **Note:** if the certificate bundle does not contain a certificate chain that verifies the Scalyr SSL certificate then all requests to Scalyr will fail unless ***ssl_verify_peer*** is set to false.  If you suspect logging to Scalyr is failing due to an invalid certificate chain, you can grep through the Fluentd output for warnings that contain the message 'certificate verification failed'.  The full text of such warnings will look something like this:
 
@@ -126,13 +134,13 @@ The cURL project maintains CA certificate bundles automatically converted from m
 
 ***message_field*** - Scalyr expects all log events to have a 'message' field containing the contents of a log message.  If your event has the log message stored in another field, you can specify the field name here, and the plugin will rename that field to 'message' before sending the data to Scalyr.  **Note:** this will override any existing 'message' field if the log record contains both a 'message' field and the field specified by this config option.
 
-***max_request_buffer*** - The maximum size in bytes of each request to send to Scalyr.  Defaults to 1,048,576 (1MB).  Fluentd chunks that generate JSON requests larger than the max_request_buffer will be split in to multiple separate requests.  **Note:** If you set this value too large Scalyr may reject your requests.
+***max_request_buffer*** - The maximum size in bytes of each request to send to Scalyr.  Defaults to 5,500,000 (5.5MB).  Fluentd chunks that generate JSON requests larger than the max_request_buffer will be split in to multiple separate requests.  **Note:** The maximum size the Scalyr servers accept for this value is 6MB and requests containing data larger than this will be rejected.
 
 ***force_message_encoding*** - Set a specific encoding for all your log messages (defaults to nil).  If your log messages are not in UTF-8, this can cause problems when converting the message to JSON in order to send to the Scalyr server.  You can avoid these problems by setting an encoding for your log messages so they can be correctly converted.
 
 ***replace_invalid_utf8*** - If this value is true and ***force_message_encoding*** is set to 'UTF-8' then all invalid UTF-8 sequences in log messages will be replaced with <?>.  Defaults to false.  This flag has no effect if ***force_message_encoding*** is not set to 'UTF-8'.
 
-####Buffer options
+### Buffer options
 
 ***retry_max_times*** - the maximum number of times to retry a failed post request before giving up.  Defaults to *40*.
 
@@ -144,7 +152,7 @@ The cURL project maintains CA certificate bundles automatically converted from m
 
 ***flush_thread_count*** - the number of threads to use to upload logs.  This is currently fixed to 1 will cause fluentd to fail with a ConfigError if set to anything greater.
 
-***chunk_limit_size*** - the maximum amount of log data to send to Scalyr in a single request.  Defaults to *100KB*.  **Note:** if you set this value too large, then Scalyr may reject your requests.  Requests smaller than 1MB will typically be accepted by Scalyr, but note that the 1MB limit also includes the entire request body and all associated JSON keys and punctuation, which may be considerably larger than the raw log data.
+***chunk_limit_size*** - the maximum amount of log data to send to Scalyr in a single request.  Defaults to *2.5MB*.  **Note:** if you set this value too large, then Scalyr may reject your requests.  Requests smaller than 6 MB will typically be accepted by Scalyr, but note that the 6 MB limit also includes the entire request body and all associated JSON keys and punctuation, which may be considerably larger than the raw log data.  This value should be set lower than the `max_request_buffer` option.
 
 ***queue_limit_length*** - the maximum number of chunks to buffer before dropping new log requests.  Defaults to *1024*.  Combines with ***chunk_limit_size*** to give you the total amount of buffer to use in the event of request failures before dropping requests.
 
@@ -172,3 +180,22 @@ Which builds the gem and puts it in the pkg directory, then install the Gem usin
 ```
 fluent-gem install pkg/fluent-plugin-scalyr-<VERSION>.gem
 ```
+
+Publishing a new release to RubyGems
+------------------------------------
+
+(for project maintainers)
+
+To publish a new version to RubyGems, simply make your changes, make sure all the lint checks and
+tests pass and merge your changes into master.
+
+After that's done, bump a version in ``VERSION`` file, update ``CHANGELOG.md`` file, add a tag
+which matches a version in VERSION file (e.g. ``v0.8.10``) and push that tag to the remote:
+
+```bash
+git tag v0.8.10
+git push origin v0.8.10
+```
+
+Push of this tag will trigger a Circle CI job which will build the latest version of the gem and
+publish it to RubyGems.

data/Rakefile

@@ -1,11 +1,15 @@
-require 'bundler'
+# frozen_string_literal: true
+
+require "bundler"
 Bundler::GemHelper.install_tasks
 
-require 'rake/testtask'
+require "rake/testtask"
 
-Rake::TestTask.new do |t|
-  t.libs << "test" << "lib"
-  t.pattern = 'test/**/test_*.rb'
+Rake::TestTask.new(:test) do |test|
+  test.libs << "lib" << "test"
+  test.test_files = FileList["test/test_*.rb"]
+  test.verbose = true
+  test.options = "--verbose=verbose"
 end
 
-task :default => [:build]
+task default: [:build]

data/VERSION

@@ -1 +1 @@
-0.8.7
+0.8.12

data/fluent-plugin-scalyr.gemspec

@@ -1,4 +1,6 @@
-$:.push File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path("lib", __dir__)
 
 Gem::Specification.new do |gem|
   gem.name = "fluent-plugin-scalyr"
@@ -9,18 +11,19 @@ Gem::Specification.new do |gem|
   gem.authors = ["Imron Alston"]
   gem.licenses = ["Apache-2.0"]
   gem.email = "imron@scalyr.com"
-  gem.has_rdoc = false
   gem.platform = Gem::Platform::RUBY
-  gem.files = Dir['AUTHORS', 'Gemfile', 'LICENSE', 'README.md', 'Rakefile', 'VERSION', 'fluent-plugin-scalyr.gemspec', 'fluent.conf.sample', 'lib/**/*', 'test/**/*']
+  gem.files = Dir["AUTHORS", "Gemfile", "LICENSE", "README.md", "Rakefile", "VERSION",
+                  "fluent-plugin-scalyr.gemspec", "fluent.conf.sample", "lib/**/*", "test/**/*"]
   gem.test_files = Dir.glob("{test,spec,features}/**/*")
-  gem.executables = Dir.glob("bin/*").map{ |f| File.basename(f) }
-  gem.require_paths = ['lib']
-  gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
+  gem.executables = Dir.glob("bin/*").map {|f| File.basename(f) }
+  gem.require_paths = ["lib"]
   gem.add_dependency "ffi", "1.9.25"
+  gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
   gem.add_dependency "rbzip2", "0.3.0"
   gem.add_dependency "zlib"
+  gem.add_development_dependency "bundler", "~> 1.9"
+  gem.add_development_dependency "flexmock", "~> 1.2"
   gem.add_development_dependency "rake", "~> 0.9"
+  gem.add_development_dependency "rubocop", "~> 0.4"
   gem.add_development_dependency "test-unit", "~> 3.0"
-  gem.add_development_dependency "flexmock", "~> 1.2"
-  gem.add_development_dependency "bundler", "~> 1.9"
 end

data/fluent.conf.sample

@@ -1,7 +1,7 @@
 <match scalyr.*>
   @type scalyr
   api_write_token YOUR_WRITE_LOGS_API_TOKEN
-  compression_type bz2
+  compression_type deflate
 
   ##Scalyr specific options
   # server_attributes {

data/lib/fluent/plugin/out_scalyr.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Scalyr Output Plugin for Fluentd
 #
@@ -15,45 +17,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-require 'fluent/plugin/output'
-require 'fluent/plugin/scalyr-exceptions'
-require 'fluent/plugin_helper/compat_parameters'
-require 'json'
-require 'net/http'
-require 'net/https'
-require 'rbzip2'
-require 'stringio'
-require 'zlib'
-require 'securerandom'
-require 'thread'
-
+require "fluent/plugin/output"
+require "fluent/plugin/scalyr_exceptions"
+require "fluent/plugin/scalyr_utils"
+require "fluent/plugin_helper/compat_parameters"
+require "json"
+require "net/http"
+require "net/https"
+require "rbzip2"
+require "stringio"
+require "zlib"
+require "securerandom"
+require "socket"
 module Scalyr
   class ScalyrOut < Fluent::Plugin::Output
-    Fluent::Plugin.register_output( 'scalyr', self )
+    Fluent::Plugin.register_output("scalyr", self)
     helpers :compat_parameters
     helpers :event_emitter
 
     config_param :api_write_token, :string
-    config_param :server_attributes, :hash, :default => nil
-    config_param :scalyr_server, :string, :default => "https://agent.scalyr.com/"
-    config_param :ssl_ca_bundle_path, :string, :default => "/etc/ssl/certs/ca-bundle.crt"
-    config_param :ssl_verify_peer, :bool, :default => true
-    config_param :ssl_verify_depth, :integer, :default => 5
-    config_param :message_field, :string, :default => "message"
-    config_param :max_request_buffer, :integer, :default => 1024*1024
-    config_param :force_message_encoding, :string, :default => nil
-    config_param :replace_invalid_utf8, :bool, :default => false
-    config_param :compression_type, :string, :default => nil #Valid options are bz2, deflate or None. Defaults to None.
-    config_param :compression_level, :integer, :default => 9 #An int containing the compression level of compression to use, from 1-9. Defaults to 9 (max)
+    config_param :server_attributes, :hash, default: nil
+    config_param :use_hostname_for_serverhost, :bool, default: true
+    config_param :scalyr_server, :string, default: "https://agent.scalyr.com/"
+    config_param :ssl_ca_bundle_path, :string, default: nil
+    config_param :ssl_verify_peer, :bool, default: true
+    config_param :ssl_verify_depth, :integer, default: 5
+    config_param :message_field, :string, default: "message"
+    config_param :max_request_buffer, :integer, default: 5_500_000
+    config_param :force_message_encoding, :string, default: nil
+    config_param :replace_invalid_utf8, :bool, default: false
+    config_param :compression_type, :string, default: nil # Valid options are bz2, deflate or None. Defaults to None.
+    config_param :compression_level, :integer, default: 6 # An int containing the compression level of compression to use, from 1-9. Defaults to 6
 
     config_section :buffer do
-      config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
-      config_set_default :retry_max_interval,  30 #wait a maximum of 30 seconds per retry
-      config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
-      config_set_default :flush_interval, 5 #default flush interval of 5 seconds
-      config_set_default :chunk_limit_size, 1024*100 #default chunk size of 100k
-      config_set_default :queue_limit_length, 1024 #default queue size of 1024
+      config_set_default :retry_max_times, 40 # try a maximum of 40 times before discarding
+      config_set_default :retry_max_interval, 30 # wait a maximum of 30 seconds per retry
+      config_set_default :retry_wait, 5 # wait a minimum of 5 seconds per retry
+      config_set_default :flush_interval, 5 # default flush interval of 5 seconds
+      config_set_default :chunk_limit_size, 2_500_000 # default chunk size of 2.5mb
+      config_set_default :queue_limit_length, 1024 # default queue size of 1024
     end
 
     # support for version 0.14.0:
@@ -65,180 +67,182 @@ module Scalyr
       true
     end
 
-    def configure( conf )
+    def multi_workers_ready?
+      true
+    end
 
-      if conf.elements('buffer').empty?
-        $log.warn "Pre 0.14.0 configuration file detected.  Please consider updating your configuration file"
+    def configure(conf)
+      if conf.elements("buffer").empty?
+        $log.warn "Pre 0.14.0 configuration file detected.  Please consider updating your configuration file" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
       end
 
-      compat_parameters_buffer( conf, default_chunk_key: '' )
+      compat_parameters_buffer(conf, default_chunk_key: "")
 
       super
 
-      if @buffer.chunk_limit_size > 1024*1024
-        $log.warn "Buffer chunk size is greater than 1Mb.  This may result in requests being rejected by Scalyr"
+      if @buffer.chunk_limit_size > 6_000_000
+        $log.warn "Buffer chunk size is greater than 6Mb.  This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
       end
 
-      if @max_request_buffer > (1024*1024*3)
-        $log.warn "Maximum request buffer > 3Mb.  This may result in requests being rejected by Scalyr"
+      if @max_request_buffer > 6_000_000
+        $log.warn "Maximum request buffer > 6Mb.  This may result in requests being rejected by Scalyr" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
       end
 
       @message_encoding = nil
-      if @force_message_encoding.to_s != ''
+      if @force_message_encoding.to_s != ""
         begin
-          @message_encoding = Encoding.find( @force_message_encoding )
+          @message_encoding = Encoding.find(@force_message_encoding)
           $log.debug "Forcing message encoding to '#{@force_message_encoding}'"
         rescue ArgumentError
           $log.warn "No encoding '#{@force_message_encoding}' found.  Ignoring"
         end
       end
 
-      #evaluate any statements in string value of the server_attributes object
+      # evaluate any statements in string value of the server_attributes object
       if @server_attributes
         new_attributes = {}
         @server_attributes.each do |key, value|
-          if value.is_a?( String )
-            m = /^\#{(.*)}$/.match( value )
-            if m
-              new_attributes[key] = eval( m[1] )
-            else
-              new_attributes[key] = value
-            end
-          end
+          next unless value.is_a?(String)
+
+          m = /^\#{(.*)}$/.match(value)
+          new_attributes[key] = if m
+                                  eval(m[1]) # rubocop:disable Security/Eval
+                                else
+                                  value
+                                end
         end
         @server_attributes = new_attributes
       end
 
-      @scalyr_server << '/' unless @scalyr_server.end_with?('/')
+      # See if we should use the hostname as the server_attributes.serverHost
+      if @use_hostname_for_serverhost
+
+        # ensure server_attributes is not nil
+        @server_attributes = {} if @server_attributes.nil?
+
+        # only set serverHost if it doesn't currently exist in server_attributes
+        # Note: Use strings rather than symbols for the key, because keys coming
+        # from the config file will be strings
+        unless @server_attributes.key? "serverHost"
+          @server_attributes["serverHost"] = Socket.gethostname
+        end
+      end
+
+      @scalyr_server << "/" unless @scalyr_server.end_with?("/")
 
       @add_events_uri = URI @scalyr_server + "addEvents"
 
       num_threads = @buffer_config.flush_thread_count
 
-      #forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
-      raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}." if num_threads > 1
+      # forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
+      if num_threads > 1
+        raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}."
+      end
     end
 
     def start
       super
-      $log.info "Scalyr Fluentd Plugin ID - #{self.plugin_id()}"
-      #Generate a session id.  This will be called once for each <match> in fluent.conf that uses scalyr
+      # Generate a session id.  This will be called once for each <match> in fluent.conf that uses scalyr
       @session = SecureRandom.uuid
 
-      @sync = Mutex.new
-      #the following variables are all under the control of the above mutex
-        @thread_ids = Hash.new #hash of tags -> id
-        @next_id = 1 #incrementing thread id for the session
-        @last_timestamp = 0 #timestamp of most recent event in nanoseconds since epoch
-
+      $log.info "Scalyr Fluentd Plugin ID id=#{plugin_id} worker=#{fluentd_worker_id} session=#{@session}" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
     end
 
-    def format( tag, time, record )
-      begin
-
-        if time.nil?
-          time = Fluent::Engine.now
-        end
-
-        # handle timestamps that are not EventTime types
-        if time.is_a?( Integer )
-          time = Fluent::EventTime.new( time )
-        elsif time.is_a?( Float )
-          components = time.divmod 1 #get integer and decimal components
-          sec = components[0].to_i
-          nsec = (components[1] * 10**9).to_i
-          time = Fluent::EventTime.new( sec, nsec )
-        end
+    def format(tag, time, record)
+      time = Fluent::Engine.now if time.nil?
+
+      # handle timestamps that are not EventTime types
+      if time.is_a?(Integer)
+        time = Fluent::EventTime.new(time)
+      elsif time.is_a?(Float)
+        components = time.divmod 1 # get integer and decimal components
+        sec = components[0].to_i
+        nsec = (components[1] * 10**9).to_i
+        time = Fluent::EventTime.new(sec, nsec)
+      end
 
-        if @message_field != "message"
-          if record.key? @message_field
-            if record.key? "message"
-              $log.warn "Overwriting log record field 'message'.  You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten."
-            end
-            record["message"] = record[@message_field]
-            record.delete( @message_field )
+      if @message_field != "message"
+        if record.key? @message_field
+          if record.key? "message"
+            $log.warn "Overwriting log record field 'message'.  You are seeing this warning because in your fluentd config file you have configured the '#{@message_field}' field to be converted to the 'message' field, but the log record already contains a field called 'message' and this is now being overwritten." # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
           end
+          record["message"] = record[@message_field]
+          record.delete(@message_field)
         end
+      end
 
-        if @message_encoding and record.key? "message" and record["message"]
-          if @replace_invalid_utf8 and @message_encoding == Encoding::UTF_8
-            record["message"] = record["message"].encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
-          else
-            record["message"].force_encoding( @message_encoding )
-          end
+      if @message_encoding && record.key?("message") && record["message"]
+        if @replace_invalid_utf8 && (@message_encoding == Encoding::UTF_8)
+          record["message"] = record["message"].encode("UTF-8", invalid: :replace, undef: :replace, replace: "<?>").force_encoding("UTF-8") # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
+        else
+          record["message"].force_encoding(@message_encoding)
         end
-        [tag, time.sec, time.nsec, record].to_msgpack
-
-      rescue JSON::GeneratorError
-        $log.warn "Unable to format message due to JSON::GeneratorError.  Record is:\n\t#{record.to_s}"
-        raise
       end
+      [tag, time.sec, time.nsec, record].to_msgpack
+    rescue JSON::GeneratorError
+      $log.warn "Unable to format message due to JSON::GeneratorError.  Record is:\n\t#{record}"
+      raise
     end
 
-    #called by fluentd when a chunk of log messages is ready
-    def write( chunk )
-      begin
-        $log.debug "Size of chunk is: #{chunk.size}"
-        requests = self.build_add_events_body( chunk )
-        $log.debug "Chunk split into #{requests.size} request(s)."
-
-        requests.each_with_index { |request, index|
-          $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
-          begin
-            response = self.post_request( @add_events_uri, request[:body] )
-            self.handle_response( response )
-          rescue OpenSSL::SSL::SSLError => e
-            if e.message.include? "certificate verify failed"
-              $log.warn "SSL certificate verification failed.  Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
-            end
-            $log.warn e.message
-            $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
-          rescue Scalyr::Client4xxError => e
-            $log.warn "4XX status code received for request #{index + 1}/#{requests.size}.  Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0,1024]}..."
+    # called by fluentd when a chunk of log messages is ready
+    def write(chunk)
+      $log.debug "Size of chunk is: #{chunk.size}"
+      requests = build_add_events_body(chunk)
+      $log.debug "Chunk split into #{requests.size} request(s)."
 
+      requests.each_with_index {|request, index|
+        $log.debug "Request #{index + 1}/#{requests.size}: #{request[:body].bytesize} bytes"
+        begin
+          response = post_request(@add_events_uri, request[:body])
+          handle_response(response)
+        rescue OpenSSL::SSL::SSLError => e
+          if e.message.include? "certificate verify failed"
+            $log.warn "SSL certificate verification failed.  Please make sure your certificate bundle is configured correctly and points to a valid file. You can configure this with the ssl_ca_bundle_path configuration option. The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'" # rubocop:disable Layout/LineLength, Lint/RedundantCopDisableDirective
           end
-        }
-
-      rescue JSON::GeneratorError
-        $log.warn "Unable to format message due to JSON::GeneratorError."
-        raise
-      end
+          $log.warn e.message
+          $log.warn "Discarding buffer chunk without retrying or logging to <secondary>"
+        rescue Scalyr::Client4xxError => e
+          $log.warn "4XX status code received for request #{index + 1}/#{requests.size}.  Discarding buffer without retrying or logging.\n\t#{response.code} - #{e.message}\n\tChunk Size: #{chunk.size}\n\tLog messages this request: #{request[:record_count]}\n\tJSON payload size: #{request[:body].bytesize}\n\tSample: #{request[:body][0, 1024]}..."
+        end
+      }
+    rescue JSON::GeneratorError
+      $log.warn "Unable to format message due to JSON::GeneratorError."
+      raise
     end
 
-
-    #explicit function to convert to nanoseconds
-    #will make things easier to maintain if/when fluentd supports higher than second resolutions
-    def to_nanos( seconds, nsec )
+    # explicit function to convert to nanoseconds
+    # will make things easier to maintain if/when fluentd supports higher than second resolutions
+    def to_nanos(seconds, nsec)
       (seconds * 10**9) + nsec
     end
 
-    #explicit function to convert to milliseconds
-    #will make things easier to maintain if/when fluentd supports higher than second resolutions
-    def to_millis( timestamp )
+    # explicit function to convert to milliseconds
+    # will make things easier to maintain if/when fluentd supports higher than second resolutions
+    def to_millis(timestamp)
       (timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
     end
 
-    def post_request( uri, body )
-
-      https = Net::HTTP.new( uri.host, uri.port )
+    def post_request(uri, body)
+      https = Net::HTTP.new(uri.host, uri.port)
       https.use_ssl = true
 
-      #verify peers to prevent potential MITM attacks
+      # verify peers to prevent potential MITM attacks
       if @ssl_verify_peer
-        https.ca_file = @ssl_ca_bundle_path
+        https.ca_file = @ssl_ca_bundle_path unless @ssl_ca_bundle_path.nil?
+        https.ssl_version = :TLSv1_2
         https.verify_mode = OpenSSL::SSL::VERIFY_PEER
         https.verify_depth = @ssl_verify_depth
       end
 
-      #use compression if enabled
+      # use compression if enabled
       encoding = nil
 
       if @compression_type
-        if @compression_type == 'deflate'
-          encoding = 'deflate'
+        if @compression_type == "deflate"
+          encoding = "deflate"
           body = Zlib::Deflate.deflate(body, @compression_level)
-        elsif @compression_type == 'bz2'
-          encoding = 'bz2'
+        elsif @compression_type == "bz2"
+          encoding = "bz2"
           io = StringIO.new
           bz2 = RBzip2.default_adapter::Compressor.new io
           bz2.write body
@@ -248,179 +252,154 @@ module Scalyr
       end
 
       post = Net::HTTP::Post.new uri.path
-      post.add_field( 'Content-Type', 'application/json' )
+      post.add_field("Content-Type", "application/json")
 
-      if @compression_type
-        post.add_field( 'Content-Encoding', encoding )
-      end
+      post.add_field("Content-Encoding", encoding) if @compression_type
 
       post.body = body
 
-      https.request( post )
-
+      https.request(post)
     end
 
-    def handle_response( response )
+    def handle_response(response)
       $log.debug "Response Code: #{response.code}"
       $log.debug "Response Body: #{response.body}"
 
-      response_hash = Hash.new
+      response_hash = {}
 
       begin
-        response_hash = JSON.parse( response.body )
-      rescue
+        response_hash = JSON.parse(response.body)
+      rescue StandardError
         response_hash["status"] = "Invalid JSON response from server"
       end
 
-      #make sure the JSON reponse has a "status" field
-      if !response_hash.key? "status"
+      # make sure the JSON reponse has a "status" field
+      unless response_hash.key? "status"
         $log.debug "JSON response does not contain status message"
         raise Scalyr::ServerError.new "JSON response does not contain status message"
       end
 
       status = response_hash["status"]
 
-      #4xx codes are handled separately
+      # 4xx codes are handled separately
       if response.code =~ /^4\d\d/
         raise Scalyr::Client4xxError.new status
       else
-        if status != "success"
+        if status != "success" # rubocop:disable Style/IfInsideElse
           if status =~ /discardBuffer/
             $log.warn "Received 'discardBuffer' message from server.  Buffer dropped."
-          elsif status =~ %r"/client/"i
+          elsif status =~ %r{/client/}i
             raise Scalyr::ClientError.new status
-          else #don't check specifically for server, we assume all non-client errors are server errors
+          else # don't check specifically for server, we assume all non-client errors are server errors
             raise Scalyr::ServerError.new status
           end
-        elsif !response.code.include? "200" #response code is a string not an int
+        elsif !response.code.include? "200" # response code is a string not an int
           raise Scalyr::ServerError
         end
       end
-
     end
 
-    def build_add_events_body( chunk )
-
-      #requests
-      requests = Array.new
+    def build_add_events_body(chunk)
+      # requests
+      requests = []
 
-      #set of unique scalyr threads for this chunk
-      current_threads = Hash.new
+      # set of unique scalyr threads for this chunk
+      current_threads = {}
 
-      #byte count
+      # byte count
       total_bytes = 0
 
-      #create a Scalyr event object for each record in the chunk
-      events = Array.new
-      chunk.msgpack_each {|(tag, sec, nsec, record)|
-
-        timestamp = self.to_nanos( sec, nsec )
+      # create a Scalyr event object for each record in the chunk
+      events = []
+      chunk.msgpack_each {|(tag, sec, nsec, record)| # rubocop:disable Metrics/BlockLength
+        timestamp = to_nanos(sec, nsec)
 
-        thread_id = 0
+        thread_id = tag
 
-        @sync.synchronize {
-          #ensure timestamp is at least 1 nanosecond greater than the last one
-          timestamp = [timestamp, @last_timestamp + 1].max
-          @last_timestamp = timestamp
-
-          #get thread id or add a new one if we haven't seen this tag before
-          if @thread_ids.key? tag
-            thread_id = @thread_ids[tag]
-          else
-            thread_id = @next_id
-            @thread_ids[tag] = thread_id
-            @next_id += 1
-          end
-        }
-
-        #then update the map of threads for this chunk
+        # then update the map of threads for this chunk
         current_threads[tag] = thread_id
 
-        #add a logfile field if one doesn't exist
-        if !record.key? "logfile"
-          record["logfile"] = "/fluentd/#{tag}"
-        end
+        # add a logfile field if one doesn't exist
+        record["logfile"] = "/fluentd/#{tag}" unless record.key? "logfile"
 
-        #append to list of events
-        event = { :thread => thread_id.to_s,
-                  :ts => timestamp,
-                  :attrs => record
-                }
+        # append to list of events
+        event = {thread: thread_id.to_s,
+                 ts:     timestamp,
+                 attrs:  record}
 
-        #get json string of event to keep track of how many bytes we are sending
+        # get json string of event to keep track of how many bytes we are sending
 
         begin
           event_json = event.to_json
         rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
-          $log.warn "#{e.class}: #{e.message}"
+          $log.warn "JSON serialization of the event failed: #{e.class}: #{e.message}"
 
           # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
-          time = Fluent::EventTime.new( sec, nsec )
+          time = Fluent::EventTime.new(sec, nsec)
           router.emit_error_event(tag, time, record, e)
 
+          # Print attribute values for debugging / troubleshooting purposes
+          $log.debug "Event attributes:"
+
           event[:attrs].each do |key, value|
-            $log.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
-            event[:attrs][key] = value.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>").force_encoding('UTF-8')
+            # NOTE: value doesn't always value.encoding attribute so we use .class which is always available
+            $log.debug "\t#{key} (#{value.class}): '#{value}'"
           end
+
+          # Recursively re-encode and sanitize potentially bad string values
+          event[:attrs] = sanitize_and_reencode_value(event[:attrs])
           event_json = event.to_json
         end
 
-        #generate new request if json size of events in the array exceed maximum request buffer size
+        # generate new request if json size of events in the array exceed maximum request buffer size
         append_event = true
         if total_bytes + event_json.bytesize > @max_request_buffer
-          #make sure we always have at least one event
-          if events.size == 0
+          # make sure we always have at least one event
+          if events.empty?
             events << event
             append_event = false
           end
-          request = self.create_request( events, current_threads )
+          request = create_request(events, current_threads)
           requests << request
 
           total_bytes = 0
-          current_threads = Hash.new
-          events = Array.new
+          current_threads = {}
+          events = []
         end
 
-        #if we haven't consumed the current event already
-        #add it to the end of our array and keep track of the json bytesize
+        # if we haven't consumed the current event already
+        # add it to the end of our array and keep track of the json bytesize
         if append_event
           events << event
           total_bytes += event_json.bytesize
         end
-
       }
 
-      #create a final request with any left over events
-      request = self.create_request( events, current_threads )
+      # create a final request with any left over events
+      request = create_request(events, current_threads)
       requests << request
-
     end
 
-    def create_request( events, current_threads )
-      #build the scalyr thread objects
-      threads = Array.new
+    def create_request(events, current_threads)
+      # build the scalyr thread objects
+      threads = []
       current_threads.each do |tag, id|
-        threads << { :id => id.to_s,
-                     :name => "Fluentd: #{tag}"
-                   }
+        threads << {id:   id.to_s,
+                    name: "Fluentd: #{tag}"}
       end
 
-      current_time = self.to_millis( Fluent::Engine.now )
+      current_time = to_millis(Fluent::Engine.now)
 
-      body = { :token => @api_write_token,
-                  :client_timestamp => current_time.to_s,
-                  :session => @session,
-                  :events => events,
-                  :threads => threads
-                }
+      body = {token:            @api_write_token,
+              client_timestamp: current_time.to_s,
+              session:          @session,
+              events:           events,
+              threads:          threads}
 
-      #add server_attributes hash if it exists
-      if @server_attributes
-        body[:sessionInfo] = @server_attributes
-      end
+      # add server_attributes hash if it exists
+      body[:sessionInfo] = @server_attributes if @server_attributes
 
-      { :body => body.to_json, :record_count => events.size }
+      {body: body.to_json, record_count: events.size}
     end
-
   end
 end