logstash-output-stackify 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ee3bea3022070998d5caf289643c7b356eeb29751f34109b5bae9520d55aa65
-  data.tar.gz: 28b8b344a668ad2072516a1f3030d713019aad36d84082aba567e2a33772efea
+  metadata.gz: adaf7be7736925de7b4a9d6af06d3a0e83079c22b4b966999560eae67ec62e7d
+  data.tar.gz: a7b120bd7de7f818acbcb9452adc4b9d2b9ef33f336a145e547f5dfe362462fa
 SHA512:
-  metadata.gz: 8d09b810c70cefde834eec5bc36c88412fc4a354c017a895ff54a0fea5c2baee4701dc3573ae331be1e0c2779813b9b4a4e8eed6be1c87d1267aa13b8f27bdd5
-  data.tar.gz: c09ab5ca029229d99f4d374c59034a801b98ab09656adcb0e8cdcab6150cb84d8136bb2106de319d313a59d6db654ad06ca2cbc80bc5dd419e773fa4c29d31b4
+  metadata.gz: 69d4a6bdca008bb34313dfcb9946c403eae443314d9001c650f9798325c43dff6458c6d679448ea197d30828006f04be73cbad19a84adffb8128464d69ace3a2
+  data.tar.gz: e9989d729f6a46c7e344b4d6d0bd60572fb051da8e25064e9dc4582ba7eacc25f4ecb4d8c67dc03403bd9e55a5d529a351ee61f2ee8d93903d0c7bcc0f951bae

data/lib/logstash/outputs/stackify.rb

@@ -1,15 +1,37 @@
 # encoding: utf-8
+
+# Core classes
 require 'date'
 require 'socket'
 require 'json'
 require 'uri'
-require "net/http"
-require "net/https"
-require "logstash/outputs/base"
+
+# Logstash classes
 require "logstash/namespace"
+require "logstash/json"
+require "logstash/outputs/base"
+require "logstash/outputs/stackify/http_client"
+
 
 # Use logstash to ship logs to Stackify
 class LogStash::Outputs::Stackify < LogStash::Outputs::Base
+  include HttpClient
+
+  CONTENT_TYPE = "application/json".freeze
+  MAX_ATTEMPTS = 3
+  METHOD = :post.freeze
+  RETRYABLE_MANTICORE_EXCEPTIONS = [
+      ::Manticore::Timeout,
+      ::Manticore::SocketException,
+      ::Manticore::ClientProtocolException,
+      ::Manticore::ResolutionFailure,
+      ::Manticore::SocketTimeout
+  ].freeze
+  RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
+  URL = "https://api.stackify.com/Log/Save".freeze
+
+  concurrency :shared
+
   config_name "stackify"
 
   # Your Stackify Activation Key
@@ -39,14 +61,27 @@ class LogStash::Outputs::Stackify < LogStash::Outputs::Base
   public
 
   def register
+    @headers = {
+        "Content-Type" => CONTENT_TYPE,
+        "Accept" => CONTENT_TYPE,
+        "X-Stackify-PV" => "V1",
+        "X-Stackify-Key" => @key.to_s
+    }
+    @url = URL
+    @logger.info("Stackify Plugin Registered")
   end
 
-  # def register
+  def multi_receive(events)
+    send_events(events)
+  end
 
-  public
+  def close
+    http_client.close
+  end
 
-  def multi_receive(events)
+  private
 
+  def send_events(events)
     begin
 
       default_level = 'info'
@@ -109,10 +144,10 @@ class LogStash::Outputs::Stackify < LogStash::Outputs::Base
           begin
             timestamp = (event.get('@timestamp').to_f * 1000).to_i
           rescue StandardError => e
-            $stdout.write('Error parsing @timestamp: ' + e.to_s)
+            @logger.warn('Error parsing @timestamp: ' + e.to_s)
           end
           if timestamp.nil?
-              timestamp = DateTime.now.strftime('%Q')
+            timestamp = DateTime.now.strftime('%Q')
           end
 
           # generate key
@@ -140,7 +175,7 @@ class LogStash::Outputs::Stackify < LogStash::Outputs::Base
           log_msg_groups[log_msg_group_key]['Msgs'].push(log_msg)
 
         rescue StandardError => e
-          $stdout.write(e.to_s)
+          @logger.warn(e.to_s)
         end
 
       end
@@ -149,31 +184,96 @@ class LogStash::Outputs::Stackify < LogStash::Outputs::Base
       log_msg_groups.each do |key, log_msg_group|
 
         begin
-
-          data = log_msg_group.to_json
-          header = {
-              'Content-Type' => 'application/json',
-              'Accept' => 'application/json',
-              'X-Stackify-PV' => 'V1',
-              'X-Stackify-Key' => @key.to_s
-          }
-
-          uri = URI.parse('https://api.stackify.com/Log/Save')
-          https = Net::HTTP.new(uri.host, uri.port)
-          https.use_ssl = true
-          req = Net::HTTP::Post.new(uri.path, initheader = header)
-          req.body = data
-          https.request(req)
-
+          post_log_msg_group(log_msg_group, 0)
         rescue StandardError => e
-          $stdout.write(e.to_s)
+          @logger.warn(e.to_s)
         end
 
       end
 
     rescue StandardError => e
-      $stdout.write(e.to_s)
+      @logger.warn(e.to_s)
+    end
+
+  end
+
+
+  def post_log_msg_group(log_msg_group, attempt)
+
+    @logger.debug("Stackify Request, Attempt " + attempt.to_s)
+
+    if attempt > MAX_ATTEMPTS
+      @logger.warn(
+          "Max attempts exceeded, dropping events",
+          :attempt => attempt
+      )
+      return false
+    end
+
+    response =
+        begin
+          body =  log_msg_group.to_json
+          http_client.post(@url, :body => body, :headers => @headers)
+        rescue Exception => e
+          if retryable_exception?(e)
+            @logger.warn(
+                "Attempt #{attempt}, retryable exception when making request",
+                :attempt => attempt,
+                :class => e.class.name,
+                :message => e.message,
+                :backtrace => e.backtrace
+            )
+            return post_log_msg_group(log_msg_group, attempt + 1)
+          else
+            @logger.error(
+                "Attempt #{attempt}, fatal exception when making request",
+                :attempt => attempt,
+                :class => e.class.name,
+                :message => e.message,
+                :backtrace => e.backtrace
+            )
+            return false
+          end
+        end
+
+    @logger.debug("Stackify Response: " + response.code.to_s)
+
+    code = response.code
+
+    if code >= 200 && code <= 299
+      true
+    elsif RETRYABLE_CODES.include?(code)
+      @logger.warn(
+          "Bad retryable response from the Stackify API",
+          :attempt => attempt,
+          :code => code
+      )
+      sleep_time = sleep_for_attempt(attempt)
+      sleep(sleep_time)
+      post_log_msg_group(log_msg_group, attempt + 1)
+    else
+      @logger.error(
+          "Bad fatal response from the Stackify API",
+          :attempt => attempt,
+          :code => code
+      )
+      false
     end
 
   end
+
+
+  def retryable_exception?(e)
+    RETRYABLE_MANTICORE_EXCEPTIONS.any? do |exception_class|
+      e.is_a?(exception_class)
+    end
+  end
+
+  def sleep_for_attempt(attempt)
+    sleep_for = attempt ** 2
+    sleep_for = sleep_for <= 60 ? sleep_for : 60
+    (sleep_for / 2) + (rand(0..sleep_for) / 2)
+  end
+
+
 end # class LogStash::Outputs::Stackify

data/lib/logstash/outputs/stackify/http_client.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+require "logstash/config/mixin"
+require "manticore"
+
+# This contains the HTTP client code within it's own namespace. It is based off
+# of the logstash-mixin-http_client plugin.
+class LogStash::Outputs::Stackify < LogStash::Outputs::Base
+  module HttpClient
+    class InvalidHTTPConfigError < StandardError; end
+
+    def self.included(base)
+      base.extend(self)
+      base.setup_http_client_config
+    end
+
+    def setup_http_client_config
+      # Timeout (in seconds) for the entire request
+      config :request_timeout, :validate => :number, :default => 60
+
+      # Timeout (in seconds) to wait for data on the socket. Default is `10s`
+      config :socket_timeout, :validate => :number, :default => 10
+
+      # Timeout (in seconds) to wait for a connection to be established. Default is `10s`
+      config :connect_timeout, :validate => :number, :default => 10
+
+      # Max number of concurrent connections. Defaults to `50`
+      config :pool_max, :validate => :number, :default => 50
+
+      # If you need to use a custom X.509 CA (.pem certs) specify the path to that here
+      config :cacert, :validate => :path
+
+      # If you'd like to use a client certificate (note, most people don't want this) set the path to the x509 cert here
+      config :client_cert, :validate => :path
+
+      # If you're using a client certificate specify the path to the encryption key here
+      config :client_key, :validate => :path
+
+      # If you need to use a custom keystore (`.jks`) specify that here. This does not work with .pem keys!
+      config :keystore, :validate => :path
+
+      # Specify the keystore password here.
+      # Note, most .jks files created with keytool require a password!
+      config :keystore_password, :validate => :password
+
+      # Specify the keystore type here. One of `JKS` or `PKCS12`. Default is `JKS`
+      config :keystore_type, :validate => :string, :default => "JKS"
+
+      # If you need to use a custom truststore (`.jks`) specify that here. This does not work with .pem certs!
+      config :truststore, :validate => :path
+
+      # Specify the truststore password here.
+      # Note, most .jks files created with keytool require a password!
+      config :truststore_password, :validate => :password
+
+      # Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
+      config :truststore_type, :validate => :string, :default => "JKS"
+
+      # If you'd like to use an HTTP proxy . This supports multiple configuration syntaxes:
+      #
+      # 1. Proxy host in form: `http://proxy.org:1234`
+      # 2. Proxy host in form: `{host => "proxy.org", port => 80, scheme => 'http', user => 'username@host', password => 'password'}`
+      # 3. Proxy host in form: `{url =>  'http://proxy.org:1234', user => 'username@host', password => 'password'}`
+      config :proxy
+    end
+
+    def build_http_client_config
+      c = {
+          connect_timeout: @connect_timeout,
+          socket_timeout: @socket_timeout,
+          request_timeout: @request_timeout,
+          follow_redirects: true,
+          automatic_retries: 1,
+          retry_non_idempotent: true,
+          check_connection_timeout: 200,
+          pool_max: @pool_max,
+          pool_max_per_route: @pool_max,
+          cookies: false,
+          keepalive: true
+      }
+
+      if @proxy
+        # Symbolize keys if necessary
+        c[:proxy] = @proxy.is_a?(Hash) ?
+                        @proxy.reduce({}) {|memo,(k,v)| memo[k.to_sym] = v; memo} :
+                        @proxy
+      end
+
+      c[:ssl] = {}
+      if @cacert
+        c[:ssl][:ca_file] = @cacert
+      end
+
+      if @truststore
+        c[:ssl].merge!(
+            :truststore => @truststore,
+            :truststore_type => @truststore_type,
+            :truststore_password => @truststore_password.value
+        )
+
+        if c[:ssl][:truststore_password].nil?
+          raise LogStash::ConfigurationError, "Truststore declared without a password! This is not valid, please set the 'truststore_password' option"
+        end
+      end
+
+      if @keystore
+        c[:ssl].merge!(
+            :keystore => @keystore,
+            :keystore_type => @keystore_type,
+            :keystore_password => @keystore_password.value
+        )
+
+        if c[:ssl][:keystore_password].nil?
+          raise LogStash::ConfigurationError, "Keystore declared without a password! This is not valid, please set the 'keystore_password' option"
+        end
+      end
+
+      if @client_cert && @client_key
+        c[:ssl][:client_cert] = @client_cert
+        c[:ssl][:client_key] = @client_key
+      elsif !!@client_cert ^ !!@client_key
+        raise InvalidHTTPConfigError, "You must specify both client_cert and client_key for an HTTP client, or neither!"
+      end
+
+      c
+    end
+
+    def http_client
+      @http_client ||= Manticore::Client.new(build_http_client_config)
+    end
+  end
+end

data/logstash-output-stackify.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-stackify'
-  s.version = '1.0.3'
+  s.version = '1.0.4'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Sends logs to Stackify"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-codec-plain'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-stackify
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Stackify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-05 00:00:00.000000000 Z
+date: 2019-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,26 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -85,6 +105,7 @@ files:
 - README.md
 - docs/DEVELOPMENT.md
 - lib/logstash/outputs/stackify.rb
+- lib/logstash/outputs/stackify/http_client.rb
 - logstash-output-stackify.gemspec
 - spec/outputs/stackify_spec.rb
 homepage: https://www.stackify.com