fluent-plugin-scalyr 0.7.4 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1fc25ee3a1190c067fd302c7ddb8ae3bb59264c
-  data.tar.gz: 28249edc864fc9d08484726d2cd5fc580359ed29
+  metadata.gz: '081b15ca4f83f95b12bb784007ce3c77c2a572df'
+  data.tar.gz: 3194d197fed3ecda3fe0d8ff1a2e037d1292159e
 SHA512:
-  metadata.gz: 9d3e6e9a2ba5979e1d8dd7dc099c0bbece63fcf88f8973ce3daec668dad8777f0ac67fdb3e4b43cdac4fde3b06c3110cefbea5acf23cd181284d60e8f9738964
-  data.tar.gz: 5a13b702c174f555d29ff89e6a65d9f29fc5a560b6f7440d25a5d466548a3ff2e9d236431e79e56174b3255d2ef9f34ed1e124b7540f92d165d8c967c26ec461
+  metadata.gz: e1b5ff7132f4ed915a8983b4bb7588895bed97a4b04accd77b1d8178e1e2ca9da1966e3eb4fc238a0573fa249584fa5ed83fecd6bb9fcc4c89b638fc09c6ee80
+  data.tar.gz: fa99e2c2fbde42d8df67103118479f5133f7bee64cb279a3665722cab373131ad12cffdab6287b9852b610279501cf1b8d74d2432fe465bb94d171dac2fc02c6

data/README.md

@@ -5,8 +5,11 @@ Scalyr output plugin for Fluentd
 version 0.12 and 0.14.
 
 The current master branch of the scalyr-fluentd plugin is compatible with
-Fluentd version 0.14 and above (including fluentd 1.0.0).  This branch is
-compatible with fluentd v0.12 and earlier.
+Fluentd version 0.14 and above, including fluentd v1.0.0.
+
+If you wish to use the scalyr-fluentd plugin with Fluentd version 0.12 or
+earlier, please use the [fluentd-0.12
+branch](https://github.com/scalyr/scalyr-fluentd/tree/cbd8c2aac3d11472121345c7cad6587a7f68c115).
 
 Overview
 --------
@@ -36,7 +39,7 @@ For example, if you want to use Scalyr's ```accessLog``` parser for all events w
 
 ```
 <filter scalyr.access>
-  type record_transformer
+  @type record_transformer
   <record>
     parser accessLog
   </record>
@@ -50,7 +53,7 @@ The Scalyr output plugin has a number of sensible defaults so the minimum config
 
 ```
 <match scalyr.*>
-  type @scalyr
+  @type scalyr
   api_write_token YOUR_SCALYR_WRITE_LOGS_TOKEN
 </match>
 ```
@@ -59,7 +62,7 @@ The following configuration options are also supported:
 
 ```
 <match scalyr.*>
-  type @scalyr
+  @type scalyr
 
   #scalyr specific options
   api_write_token YOUR_SCALYR_WRITE_TOKEN
@@ -81,13 +84,15 @@ The following configuration options are also supported:
   replace_invalid_utf8 false
 
   #buffered output options
-  retry_limit 40
-  retry_wait 5s
-  max_retry_wait 30s
-  flush_interval 5s
-  buffer_chunk_limit 100k
-  buffer_queue_limit 1024
-  num_threads 1
+  <buffer>
+    retry_max_times 40
+    retry_wait 5s
+    retry_max_interval 30s
+    flush_interval 5s
+    flush_thread_count 1
+    chunk_limit_size 100k
+    queue_limit_length 1024
+  </buffer>
 
 </match>
 ```
@@ -124,21 +129,21 @@ The cURL project maintains CA certificate bundles automatically converted from m
 
 ***replace_invalid_utf8*** - If this value is true and ***force_message_encoding*** is set to 'UTF-8' then all invalid UTF-8 sequences in log messages will be replaced with <?>.  Defaults to false.  This flag has no effect if ***force_message_encoding*** is not set to 'UTF-8'.
 
-####BufferedOutput options (inherited from Fluent::BufferedOutput)
+####Buffer options
 
-***retry_limit*** - the maximum number of times to retry a failed post request before giving up.  Defaults to *40*.
+***retry_max_times*** - the maximum number of times to retry a failed post request before giving up.  Defaults to *40*.
 
-***retry_wait*** - the initial time to wait before retrying a failed request.  Defaults to *5 seconds*.  Wait times will increase up to a maximum of ***max_retry_wait***
+***retry_wait*** - the initial time to wait before retrying a failed request.  Defaults to *5 seconds*.  Wait times will increase up to a maximum of ***retry_max_interval***
 
-***max_retry_wait*** - the maximum time to wait between retrying failed requests.  Defaults to *30 seconds*.  **Note:** This is not the total maximum time of all retry waits, but rather the maximum time to wait for a single retry.
+***retry_max_interval*** - the maximum time to wait between retrying failed requests.  Defaults to *30 seconds*.  **Note:** This is not the total maximum time of all retry waits, but rather the maximum time to wait for a single retry.
 
 ***flush_interval*** - how often to upload logs to Scalyr.  Defaults to *5 seconds*.
 
-***buffer_chunk_limit*** - the maximum amount of log data to send to Scalyr in a single request.  Defaults to *100KB*.  **Note:** if you set this value too large, then Scalyr may reject your requests.  Requests smaller than 1MB will typically be accepted by Scalyr, but note that the 1MB limit also includes the entire request body and all associated JSON keys and punctuation, which may be considerably larger than the raw log data.
+***flush_thread_count*** - the number of threads to use to upload logs.  This is currently fixed to 1 will cause fluentd to fail with a ConfigError if set to anything greater.
 
-***buffer_queue_limit*** - the maximum number of chunks to buffer before dropping new log requests.  Defaults to *1024*.  Combines with ***buffer_chunk_limit*** to give you the total amount of buffer to use in the event of request failures before dropping requests.
+***chunk_limit_size*** - the maximum amount of log data to send to Scalyr in a single request.  Defaults to *100KB*.  **Note:** if you set this value too large, then Scalyr may reject your requests.  Requests smaller than 1MB will typically be accepted by Scalyr, but note that the 1MB limit also includes the entire request body and all associated JSON keys and punctuation, which may be considerably larger than the raw log data.
 
-***num_threads*** - the number of threads to use to upload logs.  This is currently fixed to 1 will cause fluentd to fail with a ConfigError if set to anything greater.  
+***queue_limit_length*** - the maximum number of chunks to buffer before dropping new log requests.  Defaults to *1024*.  Combines with ***chunk_limit_size*** to give you the total amount of buffer to use in the event of request failures before dropping requests.
 
 Secondary Logging
 -----------------

data/VERSION

@@ -1 +1 @@
-0.7.4
+0.8.3

data/fluent-plugin-scalyr.gemspec

@@ -8,14 +8,14 @@ Gem::Specification.new do |gem|
   gem.version = File.read("VERSION").strip
   gem.authors = ["Imron Alston"]
   gem.licenses = ["Apache-2.0"]
-  gem.email = "imron@imralsoftware.com"
+  gem.email = "imron@scalyr.com"
   gem.has_rdoc = false
   gem.platform = Gem::Platform::RUBY
-  gem.files = `git ls-files`.split("\n")
-  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
-  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files = Dir['AUTHORS', 'Gemfile', 'LICENSE', 'README.md', 'Rakefile', 'VERSION', 'fluent-plugin-scalyr.gemspec', 'fluent.conf.sample', 'lib/**/*', 'test/**/*']
+  gem.test_files = Dir.glob("{test,spec,features}/**/*")
+  gem.executables = Dir.glob("bin/*").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
-  gem.add_dependency "fluentd", [">= 0.10.49", "< 0.14"]
+  gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
   gem.add_development_dependency "rake", "~> 0.9"
   gem.add_development_dependency "test-unit", "~> 3.0"
   gem.add_development_dependency "flexmock", "~> 1.2"

data/fluent.conf.sample

@@ -16,13 +16,14 @@
   # replace_invalid_utf8 false
 
 
-  ##BufferedOutput options
-  # retry_limit 40
-  # retry_wait 5s
-  # max_retry_wait 30s
-  # flush_interval 5s
-  # buffer_chunk_limit 100k
-  # buffer_queue_limit 1024
-  # num_threads 1
+  <buffer>
+  #  retry_max_times 40
+  #  retry_wait 5s
+  #  retry_max_interval 30s
+  #  flush_interval 5s
+  #  flush_thread_count 1
+  #  chunk_limit_size 100k
+  #  queue_limit_length 1024
+  </buffer>
 
 </match>

data/lib/fluent/plugin/out_scalyr.rb

@@ -16,7 +16,9 @@
 # limitations under the License.
 
 
+require 'fluent/plugin/output'
 require 'fluent/plugin/scalyr-exceptions'
+require 'fluent/plugin_helper/compat_parameters'
 require 'json'
 require 'net/http'
 require 'net/https'
@@ -24,8 +26,9 @@ require 'securerandom'
 require 'thread'
 
 module Scalyr
-  class ScalyrOut < Fluent::BufferedOutput
+  class ScalyrOut < Fluent::Plugin::Output
     Fluent::Plugin.register_output( 'scalyr', self )
+    helpers :compat_parameters
 
     config_param :api_write_token, :string
     config_param :server_attributes, :hash, :default => nil
@@ -38,23 +41,35 @@ module Scalyr
     config_param :force_message_encoding, :string, :default => nil
     config_param :replace_invalid_utf8, :bool, :default => false
 
-    config_set_default :retry_limit, 40 #try a maximum of 40 times before discarding
-    config_set_default :retry_wait, 5 #wait a minimum of 5 seconds before retrying again
-    config_set_default :max_retry_wait,  30 #wait a maximum of 30 seconds per retry
-    config_set_default :flush_interval, 5 #default flush interval of 5 seconds
+    config_section :buffer do
+      config_set_default :retry_max_times, 40 #try a maximum of 40 times before discarding
+      config_set_default :retry_max_interval,  30 #wait a maximum of 30 seconds per retry
+      config_set_default :retry_wait, 5 #wait a minimum of 5 seconds per retry
+      config_set_default :flush_interval, 5 #default flush interval of 5 seconds
+      config_set_default :chunk_limit_size, 1024*100 #default chunk size of 100k
+      config_set_default :queue_limit_length, 1024 #default queue size of 1024
+    end
+
+    # support for version 0.14.0:
+    def compat_parameters_default_chunk_key
+      ""
+    end
+
+    def formatted_to_msgpack_binary
+      true
+    end
 
     def configure( conf )
-      #need to call this before super because there doesn't seem to be any other way to
-      #set the default value for the buffer_chunk_limit, which is created and configured in super
-      if !conf.key? "buffer_chunk_limit"
-        conf["buffer_chunk_limit"] = "100k"
-      end
-      if !conf.key? "buffer_queue_limit"
-        conf["buffer_queue_limit"] = 1024
+
+      if conf.elements('buffer').empty?
+        $log.warn "Pre 0.14.0 configuration file detected.  Please consider updating your configuration file"
       end
+
+      compat_parameters_buffer( conf, default_chunk_key: '' )
+
       super
 
-      if @buffer.buffer_chunk_limit > 1024*1024
+      if @buffer.chunk_limit_size > 1024*1024
         $log.warn "Buffer chunk size is greater than 1Mb.  This may result in requests being rejected by Scalyr"
       end
 
@@ -72,12 +87,30 @@ module Scalyr
         end
       end
 
+      #evaluate any statements in string value of the server_attributes object
+      if @server_attributes
+        new_attributes = {}
+        @server_attributes.each do |key, value|
+          if value.is_a?( String )
+            m = /^\#{(.*)}$/.match( value )
+            if m
+              new_attributes[key] = eval( m[1] )
+            else
+              new_attributes[key] = value
+            end
+          end
+        end
+        @server_attributes = new_attributes
+      end
+
       @scalyr_server << '/' unless @scalyr_server.end_with?('/')
 
       @add_events_uri = URI @scalyr_server + "addEvents"
 
+      num_threads = @buffer_config.flush_thread_count
+
       #forcibly limit the number of threads to 1 for now, to ensure requests always have incrementing timestamps
-      raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{@num_threads}." if @num_threads > 1
+      raise Fluent::ConfigError, "num_threads is currently limited to 1. You specified #{num_threads}." if num_threads > 1
     end
 
     def start
@@ -90,12 +123,21 @@ module Scalyr
       #the following variables are all under the control of the above mutex
         @thread_ids = Hash.new #hash of tags -> id
         @next_id = 1 #incrementing thread id for the session
-        @last_timestamp = 0 #timestamp of most recent event
+        @last_timestamp = 0 #timestamp of most recent event in nanoseconds since epoch
 
     end
 
     def format( tag, time, record )
       begin
+
+        if time.nil?
+          time = Fluent::Engine.now
+        end
+
+        if time.is_a?( Integer )
+          time = Fluent::EventTime.new( time )
+        end
+
         if @message_field != "message"
           if record.key? @message_field
             if record.key? "message"
@@ -113,7 +155,7 @@ module Scalyr
             record["message"].force_encoding( @message_encoding )
           end
         end
-        [tag, time, record].to_msgpack
+        [tag, time.sec, time.nsec, record].to_msgpack
 
       rescue JSON::GeneratorError
         $log.warn "Unable to format message due to JSON::GeneratorError.  Record is:\n\t#{record.to_s}"
@@ -152,17 +194,16 @@ module Scalyr
     end
 
 
-
     #explicit function to convert to nanoseconds
     #will make things easier to maintain if/when fluentd supports higher than second resolutions
-    def to_nanos( seconds )
-      seconds * 10**9
+    def to_nanos( seconds, nsec )
+      (seconds * 10**9) + nsec
     end
 
     #explicit function to convert to milliseconds
     #will make things easier to maintain if/when fluentd supports higher than second resolutions
-    def to_millis( seconds )
-      seconds * 10**6
+    def to_millis( timestamp )
+      (timestamp.sec * 10**3) + (timestamp.nsec / 10**6)
     end
 
     def post_request( uri, body )
@@ -238,9 +279,10 @@ module Scalyr
 
       #create a Scalyr event object for each record in the chunk
       events = Array.new
-      chunk.msgpack_each {|(tag,time,record)|
+      chunk.msgpack_each {|(tag, sec, nsec, record)|
+
+        timestamp = self.to_nanos( sec, nsec )
 
-        timestamp = self.to_nanos( time )
         thread_id = 0
 
         @sync.synchronize {
@@ -268,7 +310,7 @@ module Scalyr
 
         #append to list of events
         event = { :thread => thread_id.to_s,
-                  :ts => timestamp.to_s,
+                  :ts => timestamp,
                   :attrs => record
                 }
 

data/test/helper.rb

@@ -27,6 +27,7 @@ module Scalyr
 
     CONFIG = %[
       api_write_token test_token
+      ssl_ca_bundle_path /etc/ssl/certs/ca-certificates.crt
     ]
 
     def create_driver( conf = CONFIG )

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.8.3
 platform: ruby
 authors:
 - Imron Alston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-16 00:00:00.000000000 Z
+date: 2018-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.49
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.49
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -87,7 +87,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.9'
 description: Sends log data collected by fluentd to Scalyr (http://www.scalyr.com)
-email: imron@imralsoftware.com
+email: imron@scalyr.com
 executables: []
 extensions: []
 extra_rdoc_files: []