semantic_logger 4.1.1 → 4.2.0

data/lib/semantic_logger/appender/sentry.rb

@@ -43,19 +43,18 @@ class SemanticLogger::Appender::Sentry < SemanticLogger::Subscriber
 
   # Send an error notification to sentry
   def log(log)
-    return false unless should_log?(log)
-    # Ignore logs coming from Ravent itself
+    # Ignore logs coming from Raven itself
     return false if log.name == 'Raven'
 
     context = formatter.call(log, self)
+    attrs                     = {
+      level: context.delete(:level),
+      extra: context
+    }
     if log.exception
       context.delete(:exception)
-      Raven.capture_exception(log.exception, context)
+      Raven.capture_exception(log.exception, attrs)
     else
-      attrs                     = {
-        level: context.delete(:level),
-        extra: context
-      }
       attrs[:extra][:backtrace] = log.backtrace if log.backtrace
       Raven.capture_message(context[:message], attrs)
     end

data/lib/semantic_logger/appender/splunk.rb

@@ -98,7 +98,7 @@ class SemanticLogger::Appender::Splunk < SemanticLogger::Subscriber
   # open the handles to resources
   def reopen
     # Connect to splunk. Connect is a synonym for creating a Service by hand and calling login.
-    self.service       = Splunk::connect(config)
+    self.service = Splunk::connect(config)
 
     # The index we are logging to
     self.service_index = service.indexes[index]
@@ -106,7 +106,6 @@ class SemanticLogger::Appender::Splunk < SemanticLogger::Subscriber
 
   # Log the message to Splunk
   def log(log)
-    return false unless should_log?(log)
     event = formatter.call(log, self)
     service_index.submit(event.delete(:message), event)
     true

data/lib/semantic_logger/appender/splunk_http.rb

@@ -73,7 +73,7 @@ class SemanticLogger::Appender::SplunkHttp < SemanticLogger::Appender::Http
     @source_type = source_type
     @index       = index
 
-    super(url:   url, compress: compress, ssl: ssl, open_timeout: 2.0, read_timeout: open_timeout, continue_timeout: continue_timeout,
+    super(url:   url, compress: compress, ssl: ssl, read_timeout: read_timeout, open_timeout: open_timeout, continue_timeout: continue_timeout,
           level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
 
     # Put splunk auth token in the header of every HTTP post.
@@ -85,12 +85,11 @@ class SemanticLogger::Appender::SplunkHttp < SemanticLogger::Appender::Http
   # For splunk format requirements see:
   #   http://dev.splunk.com/view/event-collector/SP-CAAAE6P
   def call(log, logger)
-    h = SemanticLogger::Formatters::Raw.new.call(log, logger)
-    h.delete(:time)
+    h                     = SemanticLogger::Formatters::Raw.new(time_format: :seconds).call(log, logger)
     message               = {
       source: logger.application,
       host:   logger.host,
-      time:   log.time.utc.to_f,
+      time:   h.delete(:time),
       event:  h
     }
     message[:source_type] = source_type if source_type

data/lib/semantic_logger/appender/syslog.rb

@@ -167,7 +167,7 @@ module SemanticLogger
           ::Syslog.open(application, options, facility)
         when :tcp
           # Use the local logger for @remote_syslog so errors with the remote logger can be recorded locally.
-          @tcp_client_options[:logger] = SemanticLogger::Processor.logger.clone
+          @tcp_client_options[:logger] = logger
           @tcp_client_options[:server] = "#{@server}:#{@port}"
           @remote_syslog               = Net::TCPClient.new(@tcp_client_options)
         when :udp
@@ -179,8 +179,6 @@ module SemanticLogger
 
       # Write the log using the specified protocol and server.
       def log(log)
-        return false unless should_log?(log)
-
         case @protocol
         when :syslog
           # Since the Ruby Syslog API supports sprintf format strings, double up all existing '%'

data/lib/semantic_logger/appender/tcp.rb

@@ -188,12 +188,12 @@ module SemanticLogger
                      on_connect: nil, proxy_server: nil, ssl: nil,
                      level: nil, formatter: nil, filter: nil, application: nil, host: nil, &block
       )
-        @separator                 = separator
-        @options                   = {
-          server:                 server,
-          servers:                servers,
-          policy:                 policy,
-          buffered:               buffered,
+        @separator = separator
+        @options   = {
+          server:   server,
+          servers:  servers,
+          policy:   policy,
+          buffered: buffered,
           #keepalive:              keepalive,
           connect_timeout:        connect_timeout,
           read_timeout:           read_timeout,
@@ -208,7 +208,7 @@ module SemanticLogger
         }
 
         # Use the internal logger so that errors with remote logging are only written locally.
-        Net::TCPClient.logger      = SemanticLogger::Processor.logger.clone
+        Net::TCPClient.logger      = logger
         Net::TCPClient.logger.name = 'Net::TCPClient'
 
         super(level: level, formatter: formatter, filter: filter, application: application, host: host, &block)
@@ -223,8 +223,6 @@ module SemanticLogger
 
       # Write the log using the specified protocol and server.
       def log(log)
-        return false unless should_log?(log)
-
         message = formatter.call(log, self)
         @tcp_client.retry_on_connection_failure do
           @tcp_client.write("#{message}#{separator}")

data/lib/semantic_logger/appender/udp.rb

@@ -76,8 +76,6 @@ module SemanticLogger
 
       # Write the log using the specified protocol and server.
       def log(log)
-        return false unless should_log?(log)
-
         @socket.send(formatter.call(log, self), udp_flags)
         true
       end

data/lib/semantic_logger/appender/wrapper.rb

@@ -54,8 +54,6 @@ module SemanticLogger
       #  trace entries are mapped to debug since :trace is not supported by the
       #  Ruby or Rails Loggers
       def log(log)
-        return false unless should_log?(log)
-
         @logger.send(log.level == :trace ? :debug : log.level, formatter.call(log, self))
         true
       end

data/lib/semantic_logger/base.rb

@@ -124,8 +124,16 @@ module SemanticLogger
     alias_method :benchmark, :measure
 
     # Log a thread backtrace
-    def backtrace(thread: Thread.current, level: :warn, message: 'Backtrace:', payload: nil, metric: nil, metric_amount: 1)
-      log       = Log.new(name, level)
+    def backtrace(thread: Thread.current,
+                  level: :warn,
+                  message: 'Backtrace:',
+                  payload: nil,
+                  metric: nil,
+                  metric_amount: nil)
+
+      log = Log.new(name, level)
+      return false unless meets_log_level?(log)
+
       backtrace =
         if thread == Thread.current
           Log.cleanse_backtrace
@@ -142,8 +150,10 @@ module SemanticLogger
         message << backtrace.join("\n")
       end
 
-      if log.assign(message: message, backtrace: backtrace, payload: payload, metric: metric, metric_amount: metric_amount) && should_log?(log)
+      if log.assign(message: message, backtrace: backtrace, payload: payload, metric: metric, metric_amount: metric_amount) && !filtered?(log)
         self.log(log)
+      else
+        false
       end
     end
 
@@ -232,13 +242,16 @@ module SemanticLogger
       SemanticLogger.named_tags
     end
 
-    protected
-
     # Write log data to underlying data storage
     def log(log_)
       raise NotImplementedError.new('Logging Appender must implement #log(log)')
     end
 
+    # Whether this log entry meets the criteria to be logged by this appender.
+    def should_log?(log)
+      meets_log_level?(log) && !filtered?(log)
+    end
+
     private
 
     # Initializer for Abstract Class SemanticLogger::Base
@@ -281,20 +294,21 @@ module SemanticLogger
     end
 
     # Whether to log the supplied message based on the current filter if any
-    def include_message?(log)
-      return true if @filter.nil?
+    def filtered?(log)
+      return false if @filter.nil?
 
       if @filter.is_a?(Regexp)
-        (@filter =~ log.name) != nil
+        (@filter =~ log.name) == nil
       elsif @filter.is_a?(Proc)
-        @filter.call(log) == true
+        @filter.call(log) != true
+      else
+        raise(ArgumentError, "Unrecognized semantic logger filter: #{@filter.inspect}, must be a Regexp or a Proc")
       end
     end
 
-    # Whether the log message should be logged for the current logger or appender
-    def should_log?(log)
-      # Ensure minimum log level is met, and check filter
-      (level_index <= (log.level_index || 0)) && include_message?(log)
+    # Ensure minimum log level is met
+    def meets_log_level?(log)
+      (level_index <= (log.level_index || 0))
     end
 
     # Log message at the specified level
@@ -302,12 +316,18 @@ module SemanticLogger
       log        = Log.new(name, level, index)
       should_log =
         if payload.nil? && exception.nil? && message.is_a?(Hash)
-          log.assign(message)
+          # Check if someone just logged a hash payload instead of meaning to call semantic logger
+          if message.has_key?(:message) || message.has_key?(:payload) || message.has_key?(:exception) || message.has_key?(:metric)
+            log.assign(message)
+          else
+            log.assign_positional(nil, message, nil, &block)
+          end
         else
           log.assign_positional(message, payload, exception, &block)
         end
 
-      self.log(log) if should_log && include_message?(log)
+      # Log level may change during assign due to :on_exception_level
+      self.log(log) if should_log && should_log?(log)
     end
 
     # Measure the supplied block and log the message
@@ -346,26 +366,63 @@ module SemanticLogger
           end
 
         # Extract options after block completes so that block can modify any of the options
-        payload   = params[:payload]
+        payload = params[:payload]
 
+        # May return false due to elastic logging
         should_log = log.assign(
           message:            message,
           payload:            payload,
           min_duration:       params[:min_duration] || 0.0,
           exception:          exception,
           metric:             params[:metric],
-          metric_amount:      1,
+          metric_amount:      params[:metric_amount],
           duration:           duration,
-          backtrace:          nil,
           log_exception:      params[:log_exception] || :partial,
           on_exception_level: params[:on_exception_level]
         )
 
-        self.log(log) if should_log && include_message?(log)
+        # Log level may change during assign due to :on_exception_level
+        self.log(log) if should_log && should_log?(log)
         raise exception if exception
         result
       end
     end
 
+    # For measuring methods and logging their duration.
+    def measure_method(index:,
+                       level:,
+                       message:,
+                       min_duration:,
+                       metric:,
+                       log_exception:,
+                       on_exception_level:,
+                       &block)
+
+      # Ignores filter, silence, payload
+      exception = nil
+      start     = Time.now
+      begin
+        yield
+      rescue Exception => exc
+        exception = exc
+      ensure
+        log = Log.new(name, level, index)
+        # May return false due to elastic logging
+        should_log = log.assign(
+          message:            message,
+          min_duration:       min_duration,
+          exception:          exception,
+          metric:             metric,
+          duration:           1000.0 * (Time.now - start),
+          log_exception:      log_exception,
+          on_exception_level: on_exception_level
+        )
+
+        # Log level may change during assign due to :on_exception_level
+        log(log) if should_log && should_log?(log)
+        raise exception if exception
+      end
+    end
+
   end
 end

data/lib/semantic_logger/formatters.rb

@@ -0,0 +1,37 @@
+module SemanticLogger
+  module Formatters
+    # @formatter:off
+    autoload :Base,             'semantic_logger/formatters/base'
+    autoload :Color,            'semantic_logger/formatters/color'
+    autoload :Default,          'semantic_logger/formatters/default'
+    autoload :Json,             'semantic_logger/formatters/json'
+    autoload :Raw,              'semantic_logger/formatters/raw'
+    autoload :OneLine,          'semantic_logger/formatters/one_line'
+    autoload :Signalfx,         'semantic_logger/formatters/signalfx'
+    autoload :Syslog,           'semantic_logger/formatters/syslog'
+    # @formatter:on
+
+    # Return formatter that responds to call.
+    #
+    # Supports formatter supplied as:
+    # - Symbol
+    # - Hash ( Symbol => { options })
+    # - Instance of any of SemanticLogger::Formatters
+    # - Proc
+    # - Any object that responds to :call
+    def self.factory(formatter)
+      case
+      when formatter.is_a?(Symbol)
+        SemanticLogger::Utils.constantize_symbol(formatter, 'SemanticLogger::Formatters').new
+      when formatter.is_a?(Hash) && formatter.size > 0
+        fmt, options = formatter.first
+        SemanticLogger::Utils.constantize_symbol(fmt.to_sym, 'SemanticLogger::Formatters').new(options)
+      when formatter.respond_to?(:call)
+        formatter
+      else
+        raise(ArgumentError, "Unknown formatter: #{formatter.inspect}")
+      end
+    end
+
+  end
+end

data/lib/semantic_logger/formatters/base.rb

@@ -9,7 +9,7 @@ module SemanticLogger
       PRECISION   =
         if defined?(JRuby)
           if (JRUBY_VERSION.to_f >= 9.1)
-            maint = JRUBY_VERSION.match(/\A\d\.\d\.(\d)\./)[1].to_i
+            maint = JRUBY_VERSION.match(/\A\d+\.\d+\.(\d+)\./)[1].to_i
             (maint >= 8) || (JRUBY_VERSION.to_f > 9.1) ? 6 : 3
           else
             3
@@ -21,8 +21,9 @@ module SemanticLogger
 
       # Parameters
       #   time_format: [String|Symbol|nil]
-      #     See Time#strftime for the format of this string
-      #     :iso_8601 Outputs an ISO8601 Formatted timestamp
+      #     See Time#strftime for the format of this string.
+      #     :iso_8601 Outputs an ISO8601 Formatted timestamp.
+      #     :ms       Output in miliseconds since epoch.
       #     nil:      Returns Empty string for time ( no time is output ).
       #     Default: '%Y-%m-%d %H:%M:%S.%6N'
       def initialize(time_format: TIME_FORMAT, log_host: true, log_application: true)
@@ -43,6 +44,12 @@ module SemanticLogger
         case time_format
         when :iso_8601
           time.utc.iso8601(PRECISION)
+        when :ms
+          (time.to_f * 1_000).to_i
+        when :none
+          time
+        when :seconds
+          time.to_f
         when nil
           ''
         else

data/lib/semantic_logger/formatters/json.rb

@@ -3,12 +3,8 @@ module SemanticLogger
   module Formatters
     class Json < Raw
       # Default JSON time format is ISO8601
-      def initialize(time_format: :iso_8601, log_host: true, log_application: true)
-        super(time_format: time_format, log_host: log_host, log_application: log_application)
-      end
-
-      def time
-        hash[:timestamp] = format_time(log.time)
+      def initialize(time_format: :iso_8601, log_host: true, log_application: true, time_key: :timestamp)
+        super(time_format: time_format, log_host: log_host, log_application: log_application, time_key: time_key)
       end
 
       # Returns log messages in JSON format

data/lib/semantic_logger/formatters/one_line.rb

@@ -0,0 +1,18 @@
+module SemanticLogger
+  module Formatters
+    # Only output one line for each log entry.
+    #
+    # Notes:
+    # * New lines are stripped from log messages.
+    # * Exceptions only include the class and message, the stack trace is not shown.
+    class OneLine < Default
+      def message
+        "-- #{log.message.gsub("\n", '')}" if log.message
+      end
+
+      def exception
+        "-- Exception: #{log.exception.class}: #{log.exception.message.gsub("\n", '')}" if log.exception
+      end
+    end
+  end
+end

data/lib/semantic_logger/formatters/raw.rb

@@ -4,7 +4,13 @@ module SemanticLogger
     class Raw < Base
 
       # Fields are added by populating this hash.
-      attr_accessor :hash, :log, :logger
+      attr_accessor :hash, :log, :logger, :time_key
+
+      # By default Raw formatter does not reformat the time
+      def initialize(time_format: :none, log_host: true, log_application: true, time_key: :time)
+        @time_key = time_key
+        super(time_format: time_format, log_host: log_host, log_application: log_application)
+      end
 
       # Host name
       def host
@@ -18,7 +24,7 @@ module SemanticLogger
 
       # Date & time
       def time
-        hash[:time] = log.time
+        hash[time_key] = format_time(log.time)
       end
 
       # Log level

data/lib/semantic_logger/formatters/signalfx.rb

@@ -0,0 +1,169 @@
+require 'json'
+module SemanticLogger
+  module Formatters
+    class Signalfx < Base
+      attr_accessor :token, :dimensions, :hash, :log, :logger, :gauge_name, :counter_name, :environment
+
+      def initialize(token:,
+                     dimensions: nil,
+                     log_host: true,
+                     log_application: true,
+                     gauge_name: 'Application.average',
+                     counter_name: 'Application.counter',
+                     environment: true)
+
+        @token        = token
+        @dimensions   = dimensions.map(&:to_sym) if dimensions
+        @gauge_name   = gauge_name
+        @counter_name = counter_name
+
+        if environment == true
+          @environment = defined?(Rails) ? Rails.env : ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+        elsif environment
+          @environment = environment
+        end
+
+        super(time_format: :ms, log_host: log_host, log_application: log_application)
+      end
+
+      # Create SignalFx friendly metric.
+      #   Strip leading '/'
+      #   Convert remaining '/' to '.'
+      def metric
+        if log.dimensions
+          name = log.metric.to_s.sub(/\A\/+/, '')
+          name.gsub!('/', '.')
+          hash[:metric] = name
+        else
+          # Extract class and action from metric name
+          name  = log.metric.to_s.sub(/\A\/+/, '')
+          names = name.split('/')
+          h     = (hash[:dimensions] ||= {})
+          if names.size > 1
+            h[:action] = names.pop
+            h[:class]  = names.join('::')
+          else
+            h[:class]  = 'Unknown'
+            h[:action] = names.first || log.metric
+          end
+
+          hash[:metric] = log.duration ? gauge_name : counter_name
+        end
+      end
+
+      # Date & time
+      def time
+        # 1 second resolution, represented as ms.
+        hash[:timestamp] = log.time.to_i * 1000
+      end
+
+      # Value of this metric
+      def value
+        hash[:value] = log.metric_amount || log.duration || 1
+      end
+
+      # Dimensions for this metric
+      def format_dimensions
+        h = (hash[:dimensions] ||= {})
+        if log.dimensions
+          log.dimensions.each_pair do |name, value|
+            value   = value.to_s
+            h[name] = value unless value.empty?
+          end
+        else
+          log.named_tags.each_pair do |name, value|
+            name  = name.to_sym
+            value = value.to_s
+            next if value.empty?
+            h[name] = value if dimensions && dimensions.include?(name)
+          end
+        end
+        h[:host]        = logger.host if log_host && logger.host
+        h[:application] = logger.application if log_application && logger.application
+        h[:environment] = environment if environment
+      end
+
+      # Returns [Hash] log message in Signalfx format.
+      def call(log, logger)
+        self.hash   = {}
+        self.log    = log
+        self.logger = logger
+
+        metric; time; value; format_dimensions
+
+        # gauge, counter, or cumulative_counter
+        data = {}
+        if log.duration
+          data[:gauge] = [hash]
+          # Also send a count metric whenever it is a gauge so that it can be counted.
+          unless log.dimensions
+            count_hash          = hash.dup
+            count_hash[:value]  = log.metric_amount || 1
+            count_hash[:metric] = counter_name
+            data[:counter]      = [count_hash]
+          end
+        else
+          data[:counter] = [hash]
+        end
+
+        data.to_json
+      end
+
+      # Returns [Hash] a batch of log messages.
+      # Signalfx has a minimum resolution of 1 second.
+      # Metrics of the same type, time (second), and dimensions can be aggregated together.
+      def batch(logs, logger)
+        self.logger = logger
+
+        data = {}
+        logs.each do |log|
+          self.hash = {}
+          self.log  = log
+
+          metric; time; value; format_dimensions
+
+          if log.duration
+            gauges = (data[:gauge] ||= [])
+            add_gauge(gauges, hash)
+
+            # Also send a count metric whenever it is a gauge so that it can be counted.
+            unless log.dimensions
+              count_hash          = hash.dup
+              count_hash[:value]  = log.metric_amount || 1
+              count_hash[:metric] = counter_name
+              counters            = (data[:counter] ||= [])
+              add_counter(counters, count_hash)
+            end
+          else
+            counters = (data[:counter] ||= [])
+            add_counter(counters, hash)
+          end
+        end
+
+        data.to_json
+      end
+
+      private
+
+      def add_gauge(gauges, metric)
+        gauges << metric
+      end
+
+      # Sum counters with the same time (second), name, and dimensions.
+      def add_counter(counters, metric)
+        existing = find_match(counters, metric)
+        existing ? existing[:value] += metric[:value] : counters << metric
+      end
+
+      # Find Metrics with the same timestamp, metric name, and dimensions.
+      def find_match(list, metric)
+        list.find do |item|
+          (item[:timestamp] == metric[:timestamp]) &&
+            (item[:metric] == metric[:metric]) &&
+            (item[:dimensions] == metric[:dimensions])
+        end
+      end
+
+    end
+  end
+end