logjam_agent 0.29.5 → 0.32.1

data/lib/logjam_agent/rack/logger.rb

@@ -7,14 +7,19 @@ module LogjamAgent
     class Logger < ActiveSupport::LogSubscriber
       def initialize(app, taggers = nil)
         @app = app
-        @taggers = taggers || Rails.application.config.log_tags || []
+        @taggers = taggers || (defined?(Rails::Railtie) ? Rails.application.config.log_tags : []) || []
         @hostname = LogjamAgent.hostname
         @asset_prefix = Rails.application.config.assets.prefix rescue "---"
         @ignore_asset_requests = LogjamAgent.ignore_asset_requests
       end
 
       def call(env)
-        request = ActionDispatch::Request.new(env)
+        if env["logjam_agent.framework"] == :sinatra
+          request = ::Sinatra::Request.new(env)
+          env["rack.logger"] = logger
+        else
+          request = ActionDispatch::Request.new(env)
+        end
 
         if logger.respond_to?(:tagged) && !@taggers.empty?
           logger.tagged(compute_tags(request)) { call_app(request, env) }
@@ -88,7 +93,7 @@ module LogjamAgent
         spoofed = nil
         ip = nil
         begin
-          ip = LogjamAgent.ip_obfuscator(env["action_dispatch.remote_ip"].to_s)
+          ip = LogjamAgent.ip_obfuscator((env["action_dispatch.remote_ip"] || request.ip).to_s)
         rescue ActionDispatch::RemoteIp::IpSpoofAttackError => spoofed
           ip = "*** SPOOFED IP ***"
         end
@@ -107,6 +112,11 @@ module LogjamAgent
         if completed_info = Thread.current.thread_variable_get(:time_bandits_completed_info)
           _, additions, view_time, _ = completed_info
         end
+        additions ||= []
+        if env["logjam_agent.framework"] == :sinatra
+          TimeBandits.consumed
+          additions.concat TimeBandits.runtimes
+        end
         logjam_request = LogjamAgent.request
 
         if (allowed_time_ms = env['HTTP_X_LOGJAM_CALLER_TIMEOUT'].to_i) > 0 && (run_time_ms > allowed_time_ms)
@@ -123,9 +133,9 @@ module LogjamAgent
         info message unless logjam_request.ignored?
 
         ActiveSupport::LogSubscriber.flush_all!
-        request_info = {
-          :total_time => run_time_ms, :code => status, :view_time => view_time || 0.0, :wait_time => wait_time_ms
-        }
+        request_info = { :total_time => run_time_ms, :code => status }
+        request_info[:view_time] = view_time if view_time
+        request_info[:wait_time] = wait_time_ms if wait_time_ms > 0
         logjam_request.fields.merge!(request_info)
 
         env["time_bandits.metrics"] = TimeBandits.metrics
@@ -153,7 +163,7 @@ module LogjamAgent
 
         result
       rescue Exception => e
-        Rails.logger.error(e)
+        logger.error(e)
         result
       end
 
@@ -197,82 +207,10 @@ module LogjamAgent
 
         headers
       end
-
     end
   end
 end
 
-# patch the actioncontroller logsubscriber to set the action on the logjam logger as soon as it starts processing the request
-require 'action_controller/metal/instrumentation'
-require 'action_controller/log_subscriber'
-
-module ActionController #:nodoc:
-
-  class LogSubscriber
-    if Rails::VERSION::STRING =~ /\A3\.0/
-      def start_processing(event)
-        payload = event.payload
-        params  = payload[:params].except(*INTERNAL_PARAMS)
-
-        controller = payload[:controller]
-        action = payload[:action]
-        full_name = "#{controller}##{action}"
-        action_name = LogjamAgent.action_name_proc.call(full_name)
-
-        LogjamAgent.request.fields[:action] = action_name
-
-        info "  Processing by #{full_name} as #{payload[:formats].first.to_s.upcase}"
-        info "  Parameters: #{params.inspect}" unless params.empty?
-      end
-
-    elsif Rails::VERSION::STRING =~ /\A3\.1/
-
-      def start_processing(event)
-        payload = event.payload
-        params  = payload[:params].except(*INTERNAL_PARAMS)
-        format  = payload[:format]
-        format  = format.to_s.upcase if format.is_a?(Symbol)
-
-        controller = payload[:controller]
-        action = payload[:action]
-        full_name = "#{controller}##{action}"
-        action_name = LogjamAgent.action_name_proc.call(full_name)
-
-        LogjamAgent.request.fields[:action] = action_name
-
-        info "  Processing by #{full_name} as #{format}"
-        info "  Parameters: #{params.inspect}" unless params.empty?
-      end
-
-    elsif Rails::VERSION::STRING =~ /\A(3\.2|4|5|6)/
-
-      # Rails 4.1 uses method_added to automatically subscribe newly
-      # added methods. Since start_processing is already defined, the
-      # net effect is that start_processing gets called
-      # twice. Therefore, we temporarily switch to protected mode and
-      # change it back later to public.
-      protected
-      def start_processing(event)
-        payload = event.payload
-        params  = payload[:params].except(*INTERNAL_PARAMS)
-        format  = payload[:format]
-        format  = format.to_s.upcase if format.is_a?(Symbol)
-
-        controller = payload[:controller]
-        action = payload[:action]
-        full_name = "#{controller}##{action}"
-        action_name = LogjamAgent.action_name_proc.call(full_name)
-
-        LogjamAgent.request.fields[:action] = action_name
-
-        info "Processing by #{full_name} as #{format}"
-        info "  Parameters: #{params.inspect}" unless params.empty?
-      end
-      public :start_processing
-
-    else
-      raise "logjam_agent ActionController monkey patch is not compatible with your Rails version"
-    end
-  end
-
+if defined?(Rails::Railtie)
+  require_relative "rails_support"
 end

data/lib/logjam_agent/rack/rails_support.rb

@@ -0,0 +1,26 @@
+# patch the actioncontroller logsubscriber to set the action on the logjam logger as soon as it starts processing the request
+require 'action_controller/metal/instrumentation'
+require 'action_controller/log_subscriber'
+
+module ActionController #:nodoc:
+
+  class LogSubscriber
+    def start_processing(event)
+      payload = event.payload
+      params  = payload[:params].except(*INTERNAL_PARAMS)
+      format  = payload[:format]
+      format  = format.to_s.upcase if format.is_a?(Symbol)
+
+      controller = payload[:controller]
+      action = payload[:action]
+      full_name = "#{controller}##{action}"
+      action_name = LogjamAgent.action_name_proc.call(full_name)
+
+      LogjamAgent.request.fields[:action] = action_name
+
+      info "Processing by #{full_name} as #{format}"
+      info "  Parameters: #{params.inspect}" unless params.empty?
+    end
+  end
+
+end

data/lib/logjam_agent/rack/sinatra_request.rb

@@ -0,0 +1,32 @@
+require 'sinatra/base'
+begin
+  require 'active_support/parameter_filter'
+rescue LoadError
+  require_relative '../active_support/parameter_filter'
+end
+
+# Extend the Sinatra Request class with some methods to make it look more like an
+# ActionDispatch request.
+
+class Sinatra::Request
+  alias_method :method, :request_method
+  def query_parameters; self.GET; end
+  def request_parameters; self.POST; end
+
+  def parameter_filter
+    ActiveSupport::ParameterFilter.new(LogjamAgent.parameter_filters)
+  end
+
+  KV_RE   = '[^&;=]+'
+  PAIR_RE = %r{(#{KV_RE})=(#{KV_RE})}
+
+  def filtered_path
+    return path if query_string.empty?
+    filter = parameter_filter
+    filtered_query_string = query_string.gsub(PAIR_RE) do |_|
+      filter.filter($1 => $2).first.join("=")
+    end
+    "#{path}?#{filtered_query_string}"
+  end
+
+end

data/lib/logjam_agent/railtie.rb

@@ -80,6 +80,9 @@ module LogjamAgent
       # disable logjam request forwarding by default in test environment
       LogjamAgent.disable! if Rails.env.test?
 
+      # only sent pings in production like environments
+      LogjamAgent.ensure_ping_at_exit = !%w(test development).include?(Rails.env.to_s)
+
       # patch controller testing to create a logjam request, because middlewares aren't executed
       if Rails.env.test?
         ActiveSupport.on_load(:action_controller) do

data/lib/logjam_agent/receiver.rb

@@ -0,0 +1,22 @@
+module LogjamAgent
+  class Receiver
+    def initialize
+      @socket = ZMQForwarder.context.socket(ZMQ::ROUTER)
+      @socket.setsockopt(ZMQ::RCVTIMEO, 100)
+      if @socket.bind("inproc://app") < 0
+        raise "ZMQ error on binding: #{ZMQ::Util.error_string}"
+      end
+      at_exit { @socket.close }
+    end
+
+    def receive
+      answer_parts = []
+      if @socket.recv_strings(answer_parts) < 0
+        raise "ZMQ error on receiving: #{ZMQ::Util.error_string}"
+      end
+      answer_parts.shift
+      answer_parts[2] = JSON.parse(answer_parts[2])
+      answer_parts
+    end
+  end
+end

data/lib/logjam_agent/sinatra.rb

@@ -0,0 +1,116 @@
+require 'sinatra'
+require 'logger'
+require 'logjam_agent'
+require 'logjam_agent/middleware'
+require 'logjam_agent/rack/sinatra_request'
+require 'logjam_agent/rack/logger'
+require 'time_bandits'
+
+module LogjamAgent
+  module Sinatra
+    class Middleware
+      def initialize(app)
+        app_with_logging = LogjamAgent::Rack::Logger.new(app)
+        @app = LogjamAgent::Middleware.new(app_with_logging, :sinatra)
+      end
+      def call(env)
+        @app.call(env)
+      end
+    end
+
+    module Helpers
+      def action_name(action_name)
+        LogjamAgent.request.fields[:action] = action_name
+      end
+
+      def logger
+        LogjamAgent.logger
+      end
+    end
+
+    def setup_logjam_logger
+      log_path = ENV["APP_LOG_TO_STDOUT"].present? ? STDOUT : "#{settings.root}/log/#{LogjamAgent.environment_name}.log"
+      logger = LogjamAgent::BufferedLogger.new(log_path) rescue LogjamAgent::BufferedLogger.new(STDERR)
+
+      loglevel = settings.respond_to?(:loglevel) ? settings.loglevel : :info
+      logger.level = ::Logger.const_get(loglevel.to_s.upcase)
+
+      LogjamAgent.log_device_log_level = logger.level
+      LogjamAgent.log_device_log_level = ::Logger::ERROR unless %i[test development].include?(settings.environment.to_sym)
+
+      logger.formatter = LogjamAgent::SyslogLikeFormatter.new
+      logger = ActiveSupport::TaggedLogging.new(logger)
+      LogjamAgent.logger = logger
+      ActiveSupport::LogSubscriber.logger = logger
+
+      log_path = ENV["APP_LOG_TO_STDOUT"].present? ? STDOUT : "#{settings.root}/log/logjam_agent_error.log"
+      forwarding_error_logger = ::Logger.new(log_path) rescue ::Logger.new(STDERR)
+      forwarding_error_logger.level = ::Logger::ERROR
+      forwarding_error_logger.formatter = ::Logger::Formatter.new
+      LogjamAgent.forwarding_error_logger = forwarding_error_logger
+
+      truncate_overlong_params = lambda { |key, value|
+        max_size = LogjamAgent.max_logged_size_for(key)
+        if value.is_a?(String) && value.size > max_size
+          value[max_size..-1] = " ... [TRUNCATED]"
+        end
+      }
+      LogjamAgent.parameter_filters << truncate_overlong_params
+    end
+
+    def self.registered(app)
+      app.helpers Helpers
+      LogjamAgent.environment_name = ENV['LOGJAM_ENV'] || app.settings.environment.to_s
+      LogjamAgent.auto_detect_logged_exceptions
+      LogjamAgent.disable! if app.settings.environment.to_sym == :test
+    end
+  end
+end
+
+# For classic apps.
+Sinatra.register LogjamAgent::Sinatra
+
+# We already supply a logger.
+Sinatra::Base.class_eval do
+  class << self
+    def setup_logging(builder); end
+  end
+end
+
+# Patch Sinatra's render logic to compute corrected view times.
+module LogjamAgent
+  module ComputeRenderTimes
+    def render(engine, data, options = {}, locals = {}, &block)
+      consumed_before_rendering = TimeBandits.consumed
+      result = exception = nil
+      duration = Benchmark.ms do
+        begin
+          result = super
+        rescue => exception
+        end
+      end
+      consumed_during_rendering = TimeBandits.consumed - consumed_before_rendering
+      duration -= consumed_during_rendering
+      raise exception if exception
+      result
+    ensure
+      Thread.current.thread_variable_set(
+        :time_bandits_completed_info,
+        [ duration, ["Views: %.3fms" % duration.to_f], duration, "" ]
+      )
+    end
+  end
+end
+
+Sinatra::Base.prepend LogjamAgent::ComputeRenderTimes
+
+# Define exception, but don't do anything about it. Sneaky!
+module ActionDispatch
+  module RemoteIp
+    class IpSpoofAttackError < StandardError; end
+  end
+end
+
+# Add GC time bandit
+TimeBandits.reset
+TimeBandits.add TimeBandits::TimeConsumers::GarbageCollection.instance if GC.respond_to? :enable_stats

data/lib/logjam_agent/syslog_like_formatter.rb

@@ -5,11 +5,19 @@ module LogjamAgent
     def initialize
       @hostname = LogjamAgent.hostname
       @app_name = "rails"
-      @attributes = []
       @newline = ActiveSupport::VERSION::STRING < "4.0" ? "" : "\n"
     end
 
-    attr_accessor :attributes
+    def attributes=(attributes)
+      Thread.current.thread_variable_set(:__logjam_formatter_attributes__, attributes)
+    end
+
+    def attributes
+      unless attributes = Thread.current.thread_variable_get(:__logjam_formatter_attributes__)
+        attributes = Thread.current.thread_variable_set(:__logjam_formatter_attributes__, [])
+      end
+      attributes
+    end
 
     SEV_LABEL = Logger::SEV_LABEL.map{|sev| "%-5s" % sev}
 
@@ -33,7 +41,7 @@ module LogjamAgent
       "#{format_severity(severity)} #{format_time(timestamp)}#{render_attributes}#{format_host_info(progname)}: #{format_message(msg)}#{@newline}"
     end
 
-    if !defined?(Rails) || Rails.env.development?
+    if !defined?(Rails::Railtie) || Rails.env.development?
       def format_host_info(progname); ""; end
     else
       def format_host_info(progname)
@@ -42,19 +50,19 @@ module LogjamAgent
     end
 
     def render_attributes
-      @attributes.map{|key, value| " #{key}[#{value}]"}.join
+      attributes.map{|key, value| " #{key}[#{value}]"}.join
     end
 
     def set_attribute(name, value)
-      if attribute = @attributes.detect{|n,v| n == name}
+      if attribute = attributes.detect{|n,v| n == name}
         attribute[1] = value
       else
-        @attributes << [name, value]
+        attributes << [name, value]
       end
     end
 
     def reset_attributes
-      @attributes = []
+      self.attributes = []
     end
   end
 end

data/lib/logjam_agent/util.rb

@@ -62,7 +62,12 @@ module LogjamAgent
       protocol, host, port = %r{\A(?:([^:]+)://)?([^:]+)(?::(\d+))?\z}.match(spec).captures
       protocol ||= "tcp"
       port ||= default_port
-      "#{protocol}://#{host}:#{port}"
+      if protocol == "inproc"
+        # should only be used for integration tests
+        "#{protocol}://#{host}"
+      else
+        "#{protocol}://#{host}:#{port}"
+      end
     end
   end
 end

data/lib/logjam_agent/version.rb

@@ -1,3 +1,3 @@
 module LogjamAgent
-  VERSION = "0.29.5"
+  VERSION = "0.32.1"
 end

data/lib/logjam_agent/zmq_forwarder.rb

@@ -15,7 +15,8 @@ module LogjamAgent
       @config[:host] = "localhost" if @config[:host].blank?
       @sequence = SEQUENCE_START
       @socket = nil
-      at_exit { ping; reset }
+      @ping_ensured = false
+      @socket_mutex = Mutex.new
     end
 
     def connection_specs
@@ -35,11 +36,11 @@ module LogjamAgent
       }
     end
 
-    @@mutex = Mutex.new
+    @@context_mutex = Mutex.new
     @@zmq_context = nil
 
     def self.context
-      @@mutex.synchronize do
+      @@context_mutex.synchronize do
         @@zmq_context ||=
           begin
             require 'ffi-rzmq'
@@ -50,26 +51,18 @@ module LogjamAgent
       end
     end
 
-    def socket
-      return @socket if @socket
-      @socket = self.class.context.socket(ZMQ::DEALER)
-      @socket.setsockopt(ZMQ::LINGER, @config[:linger])
-      @socket.setsockopt(ZMQ::SNDHWM, @config[:snd_hwm])
-      @socket.setsockopt(ZMQ::RCVHWM, @config[:rcv_hwm])
-      @socket.setsockopt(ZMQ::RCVTIMEO, @config[:rcv_timeo])
-      @socket.setsockopt(ZMQ::SNDTIMEO, @config[:snd_timeo])
-      spec = connection_specs.sort_by{rand}.first
-      @socket.connect(spec)
-      @socket
-    end
-
     def reset
-      if @socket
-        @socket.close
-        @socket = nil
+      @socket_mutex.synchronize do
+        reset_without_locking
       end
     end
 
+    def ensure_ping_at_exit
+      return if @ping_ensured
+      at_exit { ping; reset }
+      @ping_ensured = true
+    end
+
     def forward(data, options={})
       app_env = options[:app_env] || @app_env
       key = options[:routing_key] || "logs.#{app_env.sub('-','.')}"
@@ -77,27 +70,57 @@ module LogjamAgent
         key += ".#{engine}"
       end
       msg = LogjamAgent.encode_payload(data)
-      if options[:sync]
-        send_receive(app_env, key, msg)
-      else
-        publish(app_env, key, msg)
+      @socket_mutex.synchronize do
+        if options[:sync]
+          send_receive(app_env, key, msg)
+        else
+          publish(app_env, key, msg)
+        end
       end
     rescue => error
       reraise_expectation_errors!
       raise ForwardingError.new(error.message)
     end
 
+    private
+
+    def reset_without_locking
+      if @socket
+        @socket.close
+        @socket = nil
+      end
+    end
+
+    # this method assumes the caller holds the socket mutex
+    def socket
+      return @socket if @socket
+      @socket = self.class.context.socket(ZMQ::DEALER)
+      raise "ZMQ error on socket creation: #{ZMQ::Util.error_string}" if @socket.nil?
+      if LogjamAgent.ensure_ping_at_exit
+        ensure_ping_at_exit
+      else
+        at_exit { reset }
+      end
+      @socket.setsockopt(ZMQ::LINGER, @config[:linger])
+      @socket.setsockopt(ZMQ::SNDHWM, @config[:snd_hwm])
+      @socket.setsockopt(ZMQ::RCVHWM, @config[:rcv_hwm])
+      @socket.setsockopt(ZMQ::RCVTIMEO, @config[:rcv_timeo])
+      @socket.setsockopt(ZMQ::SNDTIMEO, @config[:snd_timeo])
+      spec = connection_specs.sort_by{rand}.first
+      @socket.connect(spec)
+      @socket
+    end
+
     def publish(app_env, key, data)
       info = pack_info(@sequence = next_fixnum(@sequence))
       parts = [app_env, key, data, info]
       if socket.send_strings(parts, ZMQ::DONTWAIT) < 0
+        error = ZMQ::Util.error_string
         reset if connection_specs.size > 1
-        raise "ZMQ error on publishing: #{ZMQ::Util.error_string}"
+        raise "ZMQ error on publishing: #{error}"
       end
     end
 
-    private
-
     def log_warning(message)
       LogjamAgent.error_handler.call ForwardingWarning.new(message)
     end
@@ -110,12 +133,12 @@ module LogjamAgent
       answer_parts = []
       if socket.send_strings(request_parts) < 0
         log_warning "ZMQ error on sending: #{ZMQ::Util.error_string}"
-        reset
+        reset_without_locking
         return nil
       end
       if socket.recv_strings(answer_parts) < 0
         log_warning "ZMQ error on receiving: #{ZMQ::Util.error_string}"
-        reset
+        reset_without_locking
         return nil
       end
       if answer_parts.first != "" || !VALID_RESPONSE_CODES.include?(answer_parts.second.to_s.to_i)
@@ -125,8 +148,10 @@ module LogjamAgent
     end
 
     def ping
-      if @socket && !send_receive("ping", @app_env, "{}", NO_COMPRESSION)
-        log_warning "failed to receive pong"
+      @socket_mutex.synchronize do
+        if @socket && !send_receive("ping", @app_env, "{}", NO_COMPRESSION)
+          log_warning "failed to receive pong"
+        end
       end
     end