fluent-plugin-sumologic_output 0.0.7 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49e311e7b4512f25f99942f69fdf4c90405d2351
-  data.tar.gz: ff095f8c1176898c8a7f97c65abc5a0c816f88db
+  metadata.gz: 5db25c9b7ac9e0a3c59424d7f3fc5c4bb1d53d27
+  data.tar.gz: fa733ca4ed03b66a475e8c06201e38584b50f5c1
 SHA512:
-  metadata.gz: 7c2c9c4a24a557cecb6e426b5069d5dfa6f6750996a262f8448cd5d79ec7192836ee1c9a0c332c93b2cd54596d48f01e0d032c8ff9b7777ca2ba939d7ba1c20e
-  data.tar.gz: 472fb4fa5312d2734704164e49b073a1b44e378690c9471c95be6f7696103f4e1daf6304c124353c6de128cd46986eda808fa91e41cecab154da1ceab5efc76c
+  metadata.gz: 36721ceacc453015d0f819c1337061d9e92c3de065f030e02f1af0802b016a6d82caa1dbb65c58bcd25e3bbcff84c8723df9c0d647cb12932b1c1cf44de994e0
+  data.tar.gz: 1dc18e8b3686efcf69794f0bf0ab55751ca89f72fc220254c4bf52279dadb066a571417b94985dd833edfc8240621061ebd03601dd2b6f0d9db6ee748828f916

data/README.md

@@ -42,6 +42,7 @@ Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
  log_format json
  source_category prod/someapp/logs
  source_name AppA
+ open_timeout 10
 </match>
 ```
 

data/fluent-plugin-sumologic_output.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sumologic_output"
-  gem.version       = "0.0.7"
+  gem.version       = "1.0.0"
   gem.authors       = ["Steven Adams"]
   gem.email         = ["stevezau@gmail.com"]
   gem.description   = %q{Output plugin to SumoLogic HTTP Endpoint}
@@ -24,4 +24,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rake"
   gem.add_runtime_dependency "fluentd"
   gem.add_development_dependency 'test-unit', '~> 3.1.0'
+
+  gem.add_runtime_dependency 'httpclient', '~> 2.8.0'
 end

data/lib/fluent/plugin/out_sumologic.rb

@@ -1,46 +1,53 @@
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'net/https'
 require 'yajl'
+require 'httpclient'
 
 class SumologicConnection
-  def initialize(endpoint, verify_ssl, open_timeout)
-    @endpoint_uri = URI.parse(endpoint.strip)
-    @verify_ssl = verify_ssl
-    @open_timeout = open_timeout
+
+  attr_reader :http
+
+  def initialize(endpoint, verify_ssl, connect_timeout)
+    @endpoint = endpoint
+    create_http_client(verify_ssl, connect_timeout)
   end
 
   def publish(raw_data, source_host=nil, source_category=nil, source_name=nil)
-    response = http.request(request_for(raw_data, source_host, source_category, source_name))
-    unless response.is_a?(Net::HTTPSuccess)
-      raise "Failed to send data to HTTP Source. #{response.code} - #{response.message}"
+    response = http.post(@endpoint, raw_data, request_headers(source_host, source_category, source_name))
+    unless response.ok?
+      raise "Failed to send data to HTTP Source. #{response.code} - #{response.body}"
     end
   end
 
   private
-  def request_for(raw_data, source_host, source_category, source_name)
-    request = Net::HTTP::Post.new(@endpoint_uri.request_uri)
-    request.body = raw_data
-    request['X-Sumo-Name'] = source_name
-    request['X-Sumo-Category'] = source_category
-    request['X-Sumo-Host'] = source_host
-    request
-  end
-
-  def http
-    # Rubys HTTP is not thread safe, so we need a new instance for each request
-    client = Net::HTTP.new(@endpoint_uri.host, @endpoint_uri.port)
-    client.use_ssl = true
-    client.verify_mode = @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
-    client.open_timeout = @open_timeout
-    client
+
+  def request_headers(source_host, source_category, source_name)
+    {
+        'X-Sumo-Name'     => source_name,
+        'X-Sumo-Category' => source_category,
+        'X-Sumo-Host'     => source_host
+    }
+  end
+
+  def ssl_options(verify_ssl)
+    verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+  end
+
+  def create_http_client(verify_ssl, connect_timeout)
+    @http                        = HTTPClient.new
+    @http.ssl_config.verify_mode = ssl_options(verify_ssl)
+    @http.connect_timeout        = connect_timeout
   end
 end
 
-class Sumologic < Fluent::BufferedOutput
+class Fluent::Plugin::Sumologic < Fluent::Plugin::Output
   # First, register the plugin. NAME is the name of this plugin
   # and identifies the plugin in the configuration file.
   Fluent::Plugin.register_output('sumologic', self)
 
+  helpers :compat_parameters
+  DEFAULT_BUFFER_TYPE = "memory"
+
   config_param :endpoint, :string
   config_param :log_format, :string, :default => 'json'
   config_param :log_key, :string, :default => 'message'
@@ -52,8 +59,20 @@ class Sumologic < Fluent::BufferedOutput
   config_param :delimiter, :string, :default => "."
   config_param :open_timeout, :integer, :default => 60
 
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+    config_set_default :chunk_keys, ['tag']
+  end
+
+  def initialize
+    super
+  end
+
   # This method is called before starting.
   def configure(conf)
+
+    compat_parameters_convert(conf, :buffer)
+
     unless conf['endpoint'] =~ URI::regexp
       raise Fluent::ConfigError, "Invalid SumoLogic endpoint url: #{conf['endpoint']}"
     end
@@ -99,7 +118,16 @@ class Sumologic < Fluent::BufferedOutput
   end
 
   def format(tag, time, record)
-    [tag, time, record].to_msgpack
+    if defined? time.nsec
+      mstime = time * 1000 + (time.nsec / 1000000)
+      [mstime, record].to_msgpack
+    else
+      [time, record].to_msgpack
+    end
+  end
+
+  def formatted_to_msgpack_binary
+    true
   end
 
   def sumo_key(sumo_metadata, record, tag)
@@ -128,13 +156,13 @@ class Sumologic < Fluent::BufferedOutput
     return param if (param =~ /\${.+}/).nil?
 
     # check for 'tag_parts[]'
-      # separated by a delimiter (default '.')
+    # separated by a delimiter (default '.')
     tag_parts = tag.split(@delimiter) unless (param =~ /tag_parts\[.+\]/).nil? || tag.nil?
 
     # pull out section between ${} then eval
     inner = param.clone
     while inner.match(/\${.+}/)
-      to_eval = inner.match(/\${(.+?)}/){$1}
+      to_eval = inner.match(/\${(.+?)}/) { $1 }
 
       if !(to_eval =~ /record\[.+\]/).nil? && record.nil?
         return to_eval
@@ -143,7 +171,7 @@ class Sumologic < Fluent::BufferedOutput
       elsif !(to_eval =~/time/).nil? && time.nil?
         return to_eval
       else
-        inner.sub!(/\${.+?}/, eval( to_eval ))
+        inner.sub!(/\${.+?}/, eval(to_eval))
       end
     end
     inner
@@ -151,16 +179,17 @@ class Sumologic < Fluent::BufferedOutput
 
   # This method is called every flush interval. Write the buffer chunk
   def write(chunk)
+    tag = chunk.metadata.tag
     messages_list = {}
 
     # Sort messages
-    chunk.msgpack_each do |tag, time, record|
+    chunk.msgpack_each do |time, record|
       # plugin dies randomly
       # https://github.com/uken/fluent-plugin-elasticsearch/commit/8597b5d1faf34dd1f1523bfec45852d380b26601#diff-ae62a005780cc730c558e3e4f47cc544R94
       next unless record.is_a? Hash
-      sumo_metadata = record.fetch('_sumo_metadata', {'source' => record[@source_name_key]})
-      key = sumo_key(sumo_metadata, record, tag)
-      log_format = sumo_metadata['log_format'] || @log_format
+      sumo_metadata = record.fetch('_sumo_metadata', { 'source' => record[@source_name_key] })
+      key           = sumo_key(sumo_metadata, record, tag)
+      log_format    = sumo_metadata['log_format'] || @log_format
 
       # Strip any unwanted newlines
       record[@log_key].chomp! if record[@log_key]
@@ -172,9 +201,9 @@ class Sumologic < Fluent::BufferedOutput
             log.strip!
           end
         when 'json_merge'
-          log = dump_log(merge_json({:timestamp => sumo_timestamp(time)}.merge(record)))
+          log = dump_log(merge_json({ :timestamp => sumo_timestamp(time) }.merge(record)))
         else
-          log = dump_log({:timestamp => sumo_timestamp(time)}.merge(record))
+          log = dump_log({ :timestamp => sumo_timestamp(time) }.merge(record))
       end
 
       unless log.nil?
@@ -192,9 +221,9 @@ class Sumologic < Fluent::BufferedOutput
       source_name, source_category, source_host = key.split(':')
       @sumo_conn.publish(
           messages.join("\n"),
-          source_host=source_host,
+          source_host    =source_host,
           source_category=source_category,
-          source_name=source_name
+          source_name    =source_name
       )
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sumologic_output
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 1.0.0
 platform: ruby
 authors:
 - Steven Adams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-26 00:00:00.000000000 Z
+date: 2017-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
 description: Output plugin to SumoLogic HTTP Endpoint
 email:
 - stevezau@gmail.com