logjam_agent 0.30.0 → 0.31.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b351167b038a91fd392f94501012260ee84319fb6035b9bf5e35811d50c885b1
-  data.tar.gz: 838094a8022bf451422bb66219924737c5d32684c7a6194c979554f101b9071e
+  metadata.gz: 6b543de9e4f7a8d5ae896a7c2fa4524a8b220a1dad8a6b4e1be6b5eee4c8110e
+  data.tar.gz: 62f2aee617a120107307e625b2ba2df8fb461f53ba9fa3baff0ce0187fc1325f
 SHA512:
-  metadata.gz: 78a136746231e26e873fcddf04740c0d748c29823e9db6fc9eabb43e9fd26cb10ec92f59eb687b70854f20a8fe918476287c171b3eca48f5006226c712ef0fb2
-  data.tar.gz: 84f8e23f8fbbc07e319e8cb928945a1c4f3198502d45a8e9a22e8dc4c9f9184f9283811c24ef40587c41d25badb6bfa3eceea7ca43b2468d3b15c1d49cb24393
+  metadata.gz: ec3578d040a1033e06029caf81256e6524a66a7c8c24493e986e467a603564e53b22cc05eadf243a3310ccfb8b3b065c012759e5d21d2acd9e64b597e4fb6710
+  data.tar.gz: ed17803ea6301229c58b82e2886cf82bb36fa88554567f5864e0d38df653a2b2a07fe76160b49de55ef03619b745608e95648364532f3f7e465021a558ae7ae1

data/README.md

@@ -5,9 +5,14 @@ Client side library for logjam.
 Hooks into Rails, collects log lines, performance metrics, error/exception infomation and Rack
 environment information and sends this data to [Logjam](https://github.com/skaes/logjam_app).
 
+Has experimental support for Sinatra.
+
 Currently only one mechanism is available for data transport:
 ZeroMQ. Support for AMQP has been dropped.
 
+[![Travis](https://travis-ci.org/skaes/logjam_agent.svg?branch=master)](https://travis-ci.org/github/skaes/logjam_agent)
+
+
 ## Usage
 
 For ZeroMQ, add
@@ -110,10 +115,8 @@ end
 
 ### Generating unique request ids
 
-The agent generates unique request ids for all request handled. It
-will use [uuid4r](https://github.com/skaes/uuid4r) if this is
-available in the application. Otherwise it will fall back to use the
-standard `SecureRandom` class shipped with Ruby.
+The agent generates unique request ids for all request handled using standard
+`SecureRandom` class shipped with Ruby.
 
 ### Generating JSON
 
@@ -121,38 +124,56 @@ The agent will try to use the [Oj](https://github.com/ohler55/oj) to
 generate JSON. If this is not available in your application, it will
 fall back to the `to_json` method.
 
-## Troubleshooting
 
-If the agent experiences problems when sending data, it will log information to a file named
-`logjam_agent_error.log` which you can find under `Rails.root/log`.
-If you set the `RAILS_LOG_TO_STDOUT` environment variable, those logs will be available through `stderr`.
+### Sinatra
 
-This behavior is customizable via a module level call back method:
+Supports both classic and modular Sinatra applications. Since Sinatra doesn't have built
+in action names like Rails, you'll have to declare them in your handlers, or in a before
+filter. Example:
 
 ```ruby
-LogjamAgent.error_handler = lambda {|exception| ... }
+require 'logjam_agent/sinatra'
+
+use LogjamAgent::Sinatra::Middleware
+
+class SinatraTestApp < Sinatra::Base
+  register LogjamAgent::Sinatra
+
+  configure do
+    set :loglevel, :debug
+    setup_logjam_logger
+
+    LogjamAgent.application_name = "myapp"
+    LogjamAgent.add_forwarder(:zmq, :host => "my-logjam-broker")
+    LogjamAgent.parameter_filters << :password
+  end
+
+  before '/index' do
+    action_name "Simple#index"
+  end
+
+  get '/index' do
+    logger.info 'Hello World!'
+    'Hello World!'
+  end
+end
 ```
 
-# License
+The environment name is picked up from either the environment variable `LOGJAM_ENV`, or
+Sinatra's environment setting.
 
-The MIT License
+Set the environment variable `APP_LOG_TO_STDOUT` if you want to log to `STDOUT`.
+Otherwise, logs will appear in the subdirectory `log` of your application's root.
 
-Copyright (c) 2013 - 2019 Stefan Kaes
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+## Troubleshooting
+
+If the agent experiences problems when sending data, it will log information to a file named
+`logjam_agent_error.log` which you can find under `Rails.root/log`.
+If you set the `RAILS_LOG_TO_STDOUT` environment variable, those logs will be available through `stderr`.
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+This behavior is customizable via a module level call back method:
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+```ruby
+LogjamAgent.error_handler = lambda {|exception| ... }
+```

data/Rakefile

@@ -7,8 +7,13 @@ Rake::TestTask.new do |t|
   t.libs << "test"
   t.test_files = FileList['test/**/*_test.rb']
   t.verbose = true
+  t.ruby_opts = %w(-W1)
 end
 
 task :default do
   Rake::Task[:test].invoke
 end
+
+task :integration do
+  sh "cd railsapp && rake"
+end

data/lib/logjam_agent.rb

@@ -36,9 +36,7 @@ require "logjam_agent/request"
 require "logjam_agent/buffered_logger"
 require "logjam_agent/syslog_like_formatter"
 
-if defined?(Rails) && Rails::VERSION::STRING >= "3.0"
-  require "logjam_agent/railtie"
-end
+require "logjam_agent/railtie" if defined?(Rails::Railtie)
 
 # monkey patch log levels to include NONE
 require 'logger'
@@ -74,6 +72,9 @@ module LogjamAgent
   mattr_accessor :action_name_proc
   self.action_name_proc = lambda{|name| name}
 
+  mattr_accessor :parameter_filters
+  self.parameter_filters = []
+
   def self.get_hostname
     n = Socket.gethostname
     if n.split('.').size > 1
@@ -97,6 +98,9 @@ module LogjamAgent
   mattr_accessor :disabled
   self.disabled = false
 
+  mattr_accessor :ensure_ping_at_exit
+  self.ensure_ping_at_exit = true
+
   mattr_accessor :obfuscate_ips
   self.obfuscate_ips = false
 

data/lib/logjam_agent/active_support/core_ext/array/extract.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Array
+  # Removes and returns the elements for which the block returns a true value.
+  # If no block is given, an Enumerator is returned instead.
+  #
+  #   numbers = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+  #   odd_numbers = numbers.extract! { |number| number.odd? } # => [1, 3, 5, 7, 9]
+  #   numbers # => [0, 2, 4, 6, 8]
+  def extract!
+    return to_enum(:extract!) { size } unless block_given?
+
+    extracted_elements = []
+
+    reject! do |element|
+      extracted_elements << element if yield(element)
+    end
+
+    extracted_elements
+  end
+end

data/lib/logjam_agent/active_support/parameter_filter.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/object/duplicable"
+require_relative "core_ext/array/extract"
+
+module ActiveSupport
+  # +ParameterFilter+ allows you to specify keys for sensitive data from
+  # hash-like object and replace corresponding value. Filtering only certain
+  # sub-keys from a hash is possible by using the dot notation:
+  # 'credit_card.number'. If a proc is given, each key and value of a hash and
+  # all sub-hashes are passed to it, where the value or the key can be replaced
+  # using String#replace or similar methods.
+  #
+  #   ActiveSupport::ParameterFilter.new([:password])
+  #   => replaces the value to all keys matching /password/i with "[FILTERED]"
+  #
+  #   ActiveSupport::ParameterFilter.new([:foo, "bar"])
+  #   => replaces the value to all keys matching /foo|bar/i with "[FILTERED]"
+  #
+  #   ActiveSupport::ParameterFilter.new(["credit_card.code"])
+  #   => replaces { credit_card: {code: "xxxx"} } with "[FILTERED]", does not
+  #   change { file: { code: "xxxx"} }
+  #
+  #   ActiveSupport::ParameterFilter.new([-> (k, v) do
+  #     v.reverse! if k =~ /secret/i
+  #   end])
+  #   => reverses the value to all keys matching /secret/i
+  class ParameterFilter
+    FILTERED = "[FILTERED]" # :nodoc:
+
+    # Create instance with given filters. Supported type of filters are +String+, +Regexp+, and +Proc+.
+    # Other types of filters are treated as +String+ using +to_s+.
+    # For +Proc+ filters, key, value, and optional original hash is passed to block arguments.
+    #
+    # ==== Options
+    #
+    # * <tt>:mask</tt> - A replaced object when filtered. Defaults to +"[FILTERED]"+
+    def initialize(filters = [], mask: FILTERED)
+      @filters = filters
+      @mask = mask
+    end
+
+    # Mask value of +params+ if key matches one of filters.
+    def filter(params)
+      compiled_filter.call(params)
+    end
+
+    # Returns filtered value for given key. For +Proc+ filters, third block argument is not populated.
+    def filter_param(key, value)
+      @filters.empty? ? value : compiled_filter.value_for_key(key, value)
+    end
+
+  private
+    def compiled_filter
+      @compiled_filter ||= CompiledFilter.compile(@filters, mask: @mask)
+    end
+
+    class CompiledFilter # :nodoc:
+      def self.compile(filters, mask:)
+        return lambda { |params| params.dup } if filters.empty?
+
+        strings, regexps, blocks = [], [], []
+
+        filters.each do |item|
+          case item
+          when Proc
+            blocks << item
+          when Regexp
+            regexps << item
+          else
+            strings << Regexp.escape(item.to_s)
+          end
+        end
+
+        deep_regexps = regexps.extract! { |r| r.to_s.include?("\\.") }
+        deep_strings = strings.extract! { |s| s.include?("\\.") }
+
+        regexps << Regexp.new(strings.join("|"), true) unless strings.empty?
+        deep_regexps << Regexp.new(deep_strings.join("|"), true) unless deep_strings.empty?
+
+        new regexps, deep_regexps, blocks, mask: mask
+      end
+
+      attr_reader :regexps, :deep_regexps, :blocks
+
+      def initialize(regexps, deep_regexps, blocks, mask:)
+        @regexps = regexps
+        @deep_regexps = deep_regexps.any? ? deep_regexps : nil
+        @blocks = blocks
+        @mask = mask
+      end
+
+      def call(params, parents = [], original_params = params)
+        filtered_params = params.class.new
+
+        params.each do |key, value|
+          filtered_params[key] = value_for_key(key, value, parents, original_params)
+        end
+
+        filtered_params
+      end
+
+      def value_for_key(key, value, parents = [], original_params = nil)
+        parents.push(key) if deep_regexps
+        if regexps.any? { |r| r.match?(key.to_s) }
+          value = @mask
+        elsif deep_regexps && (joined = parents.join(".")) && deep_regexps.any? { |r| r.match?(joined) }
+          value = @mask
+        elsif value.is_a?(Hash)
+          value = call(value, parents, original_params)
+        elsif value.is_a?(Array)
+          # If we don't pop the current parent it will be duplicated as we
+          # process each array value.
+          parents.pop if deep_regexps
+          value = value.map { |v| value_for_key(key, v, parents, original_params) }
+          # Restore the parent stack after processing the array.
+          parents.push(key) if deep_regexps
+        elsif blocks.any?
+          key = key.dup if key.duplicable?
+          value = value.dup if value.duplicable?
+          blocks.each { |b| b.arity == 2 ? b.call(key, value) : b.call(key, value, original_params) }
+        end
+        parents.pop if deep_regexps
+        value
+      end
+    end
+  end
+end

data/lib/logjam_agent/buffered_logger.rb

@@ -1,54 +1,31 @@
 require 'fileutils'
 
-if ActiveSupport::VERSION::STRING < "4.0"
-  require 'active_support/buffered_logger'
-  require 'active_support/core_ext/logger'
-  if ActiveSupport::VERSION::STRING >= "3.2"
-    require 'active_support/tagged_logging'
-    # monkey patch to handle exceptions correctly
-    # not needed for rails 4 as this uses a Formatter to add the tags
-    class ActiveSupport::TaggedLogging
-      def initialize(logger)
-        @logger = logger
-        if logger.is_a?(LogjamAgent::BufferedLogger)
-          self.class.class_eval <<-EVAL, __FILE__, __LINE__ + 1
-            def add(severity, message = nil, progname = nil, &block)
-              @logger.add(severity, message, progname, tags_text, &block)
-            end
-          EVAL
-        end
-      end
-    end
-  end
-else
-  require 'active_support/logger'
+require 'active_support/logger'
 
-  class LogjamAgent::ConsoleFormatter < Logger::Formatter
-    # This method is invoked when a log event occurs
-    def call(severity, timestamp, progname, msg)
-      "[#{format_time(timestamp)}] #{String === msg ? msg : msg.inspect}\n"
-    end
+class LogjamAgent::ConsoleFormatter < Logger::Formatter
+  # This method is invoked when a log event occurs
+  def call(severity, timestamp, progname, msg)
+    "[#{format_time(timestamp)}] #{String === msg ? msg : msg.inspect}\n"
+  end
 
-    def format_time(timestamp)
-      timestamp.strftime("%H:%M:%S.#{"%06d" % timestamp.usec}")
-    end
+  def format_time(timestamp)
+    timestamp.strftime("%H:%M:%S.#{"%06d" % timestamp.usec}")
   end
+end
 
-  class ActiveSupport::Logger
-    class << self
-      alias_method :original_broadcast, :broadcast
-      def broadcast(logger)
-        logger.formatter = LogjamAgent::ConsoleFormatter.new
-        logger.formatter.extend(ActiveSupport::TaggedLogging::Formatter)
-        original_broadcast(logger)
-      end
+class ActiveSupport::Logger
+  class << self
+    alias_method :original_broadcast, :broadcast
+    def broadcast(logger)
+      logger.formatter = LogjamAgent::ConsoleFormatter.new
+      logger.formatter.extend(ActiveSupport::TaggedLogging::Formatter)
+      original_broadcast(logger)
     end
   end
 end
 
 module LogjamAgent
-  class BufferedLogger < ( ActiveSupport::VERSION::STRING < "4.0" ?
-                           ActiveSupport::BufferedLogger : ActiveSupport::Logger )
+  class BufferedLogger < ActiveSupport::Logger
 
     attr_accessor :formatter
 
@@ -57,8 +34,6 @@ module LogjamAgent
 
     def initialize(*args)
       super(*args)
-      # stupid bug in the buffered logger code (Rails::VERSION::STRING < "3.2")
-      @log.write "\n" if @log && respond_to?(:buffer)
       @formatter = lambda{|_, _, _, message| message}
     end
 

data/lib/logjam_agent/middleware.rb

@@ -1,11 +1,16 @@
 module LogjamAgent
   class Middleware
-    def initialize(app, options={})
+    def initialize(app, framework = :rails)
       @app = app
-      @options = options
+      @framework = framework
+      unless %i{rails sinatra}.include?(framework)
+        raise ArgumentError.new("Invalid logjam_agent framework: #{framework}. Only :rails and :sinatra are valid!")
+      end
+      @reraise = defined?(Rails::Railtie) && Rails.env.test?
     end
 
     def call(env)
+      env["logjam_agent.framework"] = @framework
       strip_encoding_from_etag(env)
       request = start_request(env)
       result = @app.call(env)
@@ -13,9 +18,13 @@ module LogjamAgent
       result
     rescue Exception
       result = [500, {'Content-Type' => 'text/html'}, ["<html><body><h1>500 Internal Server Error</h1></body></html>"]]
+      raise if @reraise
     ensure
       headers = result[1]
       headers["X-Logjam-Request-Id"] = request.id
+      if env["sinatra.static_file"]
+        request.fields[:action] = "Sinatra#static_file"
+      end
       unless (request_action = request.fields[:action]).blank?
         headers["X-Logjam-Request-Action"] = request_action
       end
@@ -41,7 +50,10 @@ module LogjamAgent
       env_name      = env["logjam_agent.environment_name"] || LogjamAgent.environment_name
       caller_id     = env["HTTP_X_LOGJAM_CALLER_ID"] || ""
       caller_action = env["HTTP_X_LOGJAM_ACTION"] || ""
-      LogjamAgent.start_request(app_name, env_name, :caller_id => caller_id, :caller_action => caller_action)
+      extra_fields = {}
+      extra_fields[:caller_id] = caller_id if caller_id.present?
+      extra_fields[:caller_action] = caller_action if caller_action.present?
+      LogjamAgent.start_request(app_name, env_name, extra_fields)
     end
 
     def finish_request(env)

data/lib/logjam_agent/rack/logger.rb

@@ -7,14 +7,19 @@ module LogjamAgent
     class Logger < ActiveSupport::LogSubscriber
       def initialize(app, taggers = nil)
         @app = app
-        @taggers = taggers || Rails.application.config.log_tags || []
+        @taggers = taggers || (defined?(Rails::Railtie) ? Rails.application.config.log_tags : []) || []
         @hostname = LogjamAgent.hostname
         @asset_prefix = Rails.application.config.assets.prefix rescue "---"
         @ignore_asset_requests = LogjamAgent.ignore_asset_requests
       end
 
       def call(env)
-        request = ActionDispatch::Request.new(env)
+        if env["logjam_agent.framework"] == :sinatra
+          request = ::Sinatra::Request.new(env)
+          env["rack.logger"] = logger
+        else
+          request = ActionDispatch::Request.new(env)
+        end
 
         if logger.respond_to?(:tagged) && !@taggers.empty?
           logger.tagged(compute_tags(request)) { call_app(request, env) }
@@ -88,7 +93,7 @@ module LogjamAgent
         spoofed = nil
         ip = nil
         begin
-          ip = LogjamAgent.ip_obfuscator(env["action_dispatch.remote_ip"].to_s)
+          ip = LogjamAgent.ip_obfuscator((env["action_dispatch.remote_ip"] || request.ip).to_s)
         rescue ActionDispatch::RemoteIp::IpSpoofAttackError => spoofed
           ip = "*** SPOOFED IP ***"
         end
@@ -107,6 +112,11 @@ module LogjamAgent
         if completed_info = Thread.current.thread_variable_get(:time_bandits_completed_info)
           _, additions, view_time, _ = completed_info
         end
+        additions ||= []
+        if env["logjam_agent.framework"] == :sinatra
+          TimeBandits.consumed
+          additions.concat TimeBandits.runtimes
+        end
         logjam_request = LogjamAgent.request
 
         if (allowed_time_ms = env['HTTP_X_LOGJAM_CALLER_TIMEOUT'].to_i) > 0 && (run_time_ms > allowed_time_ms)
@@ -123,9 +133,9 @@ module LogjamAgent
         info message unless logjam_request.ignored?
 
         ActiveSupport::LogSubscriber.flush_all!
-        request_info = {
-          :total_time => run_time_ms, :code => status, :view_time => view_time || 0.0, :wait_time => wait_time_ms
-        }
+        request_info = { :total_time => run_time_ms, :code => status }
+        request_info[:view_time] = view_time if view_time
+        request_info[:wait_time] = wait_time_ms if wait_time_ms > 0
         logjam_request.fields.merge!(request_info)
 
         env["time_bandits.metrics"] = TimeBandits.metrics
@@ -153,7 +163,7 @@ module LogjamAgent
 
         result
       rescue Exception => e
-        Rails.logger.error(e)
+        logger.error(e)
         result
       end
 
@@ -197,82 +207,10 @@ module LogjamAgent
 
         headers
       end
-
     end
   end
 end
 
-# patch the actioncontroller logsubscriber to set the action on the logjam logger as soon as it starts processing the request
-require 'action_controller/metal/instrumentation'
-require 'action_controller/log_subscriber'
-
-module ActionController #:nodoc:
-
-  class LogSubscriber
-    if Rails::VERSION::STRING =~ /\A3\.0/
-      def start_processing(event)
-        payload = event.payload
-        params  = payload[:params].except(*INTERNAL_PARAMS)
-
-        controller = payload[:controller]
-        action = payload[:action]
-        full_name = "#{controller}##{action}"
-        action_name = LogjamAgent.action_name_proc.call(full_name)
-
-        LogjamAgent.request.fields[:action] = action_name
-
-        info "  Processing by #{full_name} as #{payload[:formats].first.to_s.upcase}"
-        info "  Parameters: #{params.inspect}" unless params.empty?
-      end
-
-    elsif Rails::VERSION::STRING =~ /\A3\.1/
-
-      def start_processing(event)
-        payload = event.payload
-        params  = payload[:params].except(*INTERNAL_PARAMS)
-        format  = payload[:format]
-        format  = format.to_s.upcase if format.is_a?(Symbol)
-
-        controller = payload[:controller]
-        action = payload[:action]
-        full_name = "#{controller}##{action}"
-        action_name = LogjamAgent.action_name_proc.call(full_name)
-
-        LogjamAgent.request.fields[:action] = action_name
-
-        info "  Processing by #{full_name} as #{format}"
-        info "  Parameters: #{params.inspect}" unless params.empty?
-      end
-
-    elsif Rails::VERSION::STRING =~ /\A(3\.2|4|5|6)/
-
-      # Rails 4.1 uses method_added to automatically subscribe newly
-      # added methods. Since start_processing is already defined, the
-      # net effect is that start_processing gets called
-      # twice. Therefore, we temporarily switch to protected mode and
-      # change it back later to public.
-      protected
-      def start_processing(event)
-        payload = event.payload
-        params  = payload[:params].except(*INTERNAL_PARAMS)
-        format  = payload[:format]
-        format  = format.to_s.upcase if format.is_a?(Symbol)
-
-        controller = payload[:controller]
-        action = payload[:action]
-        full_name = "#{controller}##{action}"
-        action_name = LogjamAgent.action_name_proc.call(full_name)
-
-        LogjamAgent.request.fields[:action] = action_name
-
-        info "Processing by #{full_name} as #{format}"
-        info "  Parameters: #{params.inspect}" unless params.empty?
-      end
-      public :start_processing
-
-    else
-      raise "logjam_agent ActionController monkey patch is not compatible with your Rails version"
-    end
-  end
-
+if defined?(Rails::Railtie)
+  require_relative "rails_support"
 end

data/lib/logjam_agent/rack/rails_support.rb

@@ -0,0 +1,26 @@
+# patch the actioncontroller logsubscriber to set the action on the logjam logger as soon as it starts processing the request
+require 'action_controller/metal/instrumentation'
+require 'action_controller/log_subscriber'
+
+module ActionController #:nodoc:
+
+  class LogSubscriber
+    def start_processing(event)
+      payload = event.payload
+      params  = payload[:params].except(*INTERNAL_PARAMS)
+      format  = payload[:format]
+      format  = format.to_s.upcase if format.is_a?(Symbol)
+
+      controller = payload[:controller]
+      action = payload[:action]
+      full_name = "#{controller}##{action}"
+      action_name = LogjamAgent.action_name_proc.call(full_name)
+
+      LogjamAgent.request.fields[:action] = action_name
+
+      info "Processing by #{full_name} as #{format}"
+      info "  Parameters: #{params.inspect}" unless params.empty?
+    end
+  end
+
+end

data/lib/logjam_agent/rack/sinatra_request.rb

@@ -0,0 +1,32 @@
+require 'sinatra/base'
+begin
+  require 'active_support/parameter_filter'
+rescue LoadError
+  require_relative '../active_support/parameter_filter'
+end
+
+# Extend the Sinatra Request class with some methods to make it look more like an
+# ActionDispatch request.
+
+class Sinatra::Request
+  alias_method :method, :request_method
+  def query_parameters; self.GET; end
+  def request_parameters; self.POST; end
+
+  def parameter_filter
+    ActiveSupport::ParameterFilter.new(LogjamAgent.parameter_filters)
+  end
+
+  KV_RE   = '[^&;=]+'
+  PAIR_RE = %r{(#{KV_RE})=(#{KV_RE})}
+
+  def filtered_path
+    return path if query_string.empty?
+    filter = parameter_filter
+    filtered_query_string = query_string.gsub(PAIR_RE) do |_|
+      filter.filter($1 => $2).first.join("=")
+    end
+    "#{path}?#{filtered_query_string}"
+  end
+
+end

data/lib/logjam_agent/railtie.rb

@@ -80,6 +80,9 @@ module LogjamAgent
       # disable logjam request forwarding by default in test environment
       LogjamAgent.disable! if Rails.env.test?
 
+      # only sent pings in production like environments
+      LogjamAgent.ensure_ping_at_exit = !%w(test development).include?(Rails.env.to_s)
+
       # patch controller testing to create a logjam request, because middlewares aren't executed
       if Rails.env.test?
         ActiveSupport.on_load(:action_controller) do

data/lib/logjam_agent/receiver.rb

@@ -0,0 +1,22 @@
+module LogjamAgent
+  class Receiver
+    def initialize
+      @socket = ZMQForwarder.context.socket(ZMQ::ROUTER)
+      @socket.setsockopt(ZMQ::RCVTIMEO, 100)
+      if @socket.bind("inproc://app") < 0
+        raise "ZMQ error on binding: #{ZMQ::Util.error_string}"
+      end
+      at_exit { @socket.close }
+    end
+
+    def receive
+      answer_parts = []
+      if @socket.recv_strings(answer_parts) < 0
+        raise "ZMQ error on receiving: #{ZMQ::Util.error_string}"
+      end
+      answer_parts.shift
+      answer_parts[2] = JSON.parse(answer_parts[2])
+      answer_parts
+    end
+  end
+end

data/lib/logjam_agent/sinatra.rb

@@ -0,0 +1,116 @@
+require 'sinatra'
+require 'logger'
+require 'logjam_agent'
+require 'logjam_agent/middleware'
+require 'logjam_agent/rack/sinatra_request'
+require 'logjam_agent/rack/logger'
+require 'time_bandits'
+
+module LogjamAgent
+  module Sinatra
+    class Middleware
+      def initialize(app)
+        app_with_logging = LogjamAgent::Rack::Logger.new(app)
+        @app = LogjamAgent::Middleware.new(app_with_logging, :sinatra)
+      end
+      def call(env)
+        @app.call(env)
+      end
+    end
+
+    module Helpers
+      def action_name(action_name)
+        LogjamAgent.request.fields[:action] = action_name
+      end
+
+      def logger
+        LogjamAgent.logger
+      end
+    end
+
+    def setup_logjam_logger
+      log_path = ENV["APP_LOG_TO_STDOUT"].present? ? STDOUT : "#{settings.root}/log/#{LogjamAgent.environment_name}.log"
+      logger = LogjamAgent::BufferedLogger.new(log_path) rescue LogjamAgent::BufferedLogger.new(STDERR)
+
+      loglevel = settings.respond_to?(:loglevel) ? settings.loglevel : :info
+      logger.level = ::Logger.const_get(loglevel.to_s.upcase)
+
+      LogjamAgent.log_device_log_level = logger.level
+      LogjamAgent.log_device_log_level = ::Logger::ERROR unless %i[test development].include?(settings.environment.to_sym)
+
+      logger.formatter = LogjamAgent::SyslogLikeFormatter.new
+      logger = ActiveSupport::TaggedLogging.new(logger)
+      LogjamAgent.logger = logger
+      ActiveSupport::LogSubscriber.logger = logger
+
+      log_path = ENV["APP_LOG_TO_STDOUT"].present? ? STDOUT : "#{settings.root}/log/logjam_agent_error.log"
+      forwarding_error_logger = ::Logger.new(log_path) rescue ::Logger.new(STDERR)
+      forwarding_error_logger.level = ::Logger::ERROR
+      forwarding_error_logger.formatter = ::Logger::Formatter.new
+      LogjamAgent.forwarding_error_logger = forwarding_error_logger
+
+      truncate_overlong_params = lambda { |key, value|
+        max_size = LogjamAgent.max_logged_size_for(key)
+        if value.is_a?(String) && value.size > max_size
+          value[max_size..-1] = " ... [TRUNCATED]"
+        end
+      }
+      LogjamAgent.parameter_filters << truncate_overlong_params
+    end
+
+    def self.registered(app)
+      app.helpers Helpers
+      LogjamAgent.environment_name = ENV['LOGJAM_ENV'] || app.settings.environment.to_s
+      LogjamAgent.auto_detect_logged_exceptions
+      LogjamAgent.disable! if app.settings.environment.to_sym == :test
+    end
+  end
+end
+
+# For classic apps.
+Sinatra.register LogjamAgent::Sinatra
+
+# We already supply a logger.
+Sinatra::Base.class_eval do
+  class << self
+    def setup_logging(builder); end
+  end
+end
+
+# Patch Sinatra's render logic to compute corrected view times.
+module LogjamAgent
+  module ComputeRenderTimes
+    def render(engine, data, options = {}, locals = {}, &block)
+      consumed_before_rendering = TimeBandits.consumed
+      result = exception = nil
+      duration = Benchmark.ms do
+        begin
+          result = super
+        rescue => exception
+        end
+      end
+      consumed_during_rendering = TimeBandits.consumed - consumed_before_rendering
+      duration -= consumed_during_rendering
+      raise exception if exception
+      result
+    ensure
+      Thread.current.thread_variable_set(
+        :time_bandits_completed_info,
+        [ duration, ["Views: %.3fms" % duration.to_f], duration, "" ]
+      )
+    end
+  end
+end
+
+Sinatra::Base.prepend LogjamAgent::ComputeRenderTimes
+
+# Define exception, but don't do anything about it. Sneaky!
+module ActionDispatch
+  module RemoteIp
+    class IpSpoofAttackError < StandardError; end
+  end
+end
+
+# Add GC time bandit
+TimeBandits.reset
+TimeBandits.add TimeBandits::TimeConsumers::GarbageCollection.instance if GC.respond_to? :enable_stats

data/lib/logjam_agent/syslog_like_formatter.rb

@@ -41,7 +41,7 @@ module LogjamAgent
       "#{format_severity(severity)} #{format_time(timestamp)}#{render_attributes}#{format_host_info(progname)}: #{format_message(msg)}#{@newline}"
     end
 
-    if !defined?(Rails) || Rails.env.development?
+    if !defined?(Rails::Railtie) || Rails.env.development?
       def format_host_info(progname); ""; end
     else
       def format_host_info(progname)

data/lib/logjam_agent/util.rb

@@ -62,7 +62,12 @@ module LogjamAgent
       protocol, host, port = %r{\A(?:([^:]+)://)?([^:]+)(?::(\d+))?\z}.match(spec).captures
       protocol ||= "tcp"
       port ||= default_port
-      "#{protocol}://#{host}:#{port}"
+      if protocol == "inproc"
+        # should only be used for integration tests
+        "#{protocol}://#{host}"
+      else
+        "#{protocol}://#{host}:#{port}"
+      end
     end
   end
 end

data/lib/logjam_agent/version.rb

@@ -1,3 +1,3 @@
 module LogjamAgent
-  VERSION = "0.30.0"
+  VERSION = "0.31.0"
 end

data/lib/logjam_agent/zmq_forwarder.rb

@@ -53,7 +53,12 @@ module LogjamAgent
     def socket
       return @socket if @socket
       @socket = self.class.context.socket(ZMQ::DEALER)
-      ensure_ping_at_exit
+      raise "ZMQ error on socket creation: #{ZMQ::Util.error_string}" if @socket.nil?
+      if LogjamAgent.ensure_ping_at_exit
+        ensure_ping_at_exit
+      else
+        at_exit { reset }
+      end
       @socket.setsockopt(ZMQ::LINGER, @config[:linger])
       @socket.setsockopt(ZMQ::SNDHWM, @config[:snd_hwm])
       @socket.setsockopt(ZMQ::RCVHWM, @config[:rcv_hwm])
@@ -98,8 +103,9 @@ module LogjamAgent
       info = pack_info(@sequence = next_fixnum(@sequence))
       parts = [app_env, key, data, info]
       if socket.send_strings(parts, ZMQ::DONTWAIT) < 0
+        error = ZMQ::Util.error_string
         reset if connection_specs.size > 1
-        raise "ZMQ error on publishing: #{ZMQ::Util.error_string}"
+        raise "ZMQ error on publishing: #{error}"
       end
     end
 

data/test/sinatra_app.rb

@@ -0,0 +1,32 @@
+$:.unshift File.expand_path('../../lib', __FILE__)
+
+require 'logjam_agent/sinatra'
+
+class SinatraTestApp < Sinatra::Base
+  register LogjamAgent::Sinatra
+
+  use LogjamAgent::Sinatra::Middleware
+
+  configure do
+    set :root, File.expand_path('../..', __FILE__)
+    set :environment, :test
+    set :loglevel, :debug
+    setup_logjam_logger
+
+    LogjamAgent.application_name = "myapp"
+    LogjamAgent.add_forwarder(:zmq, :host => "inproc://app")
+    LogjamAgent.parameter_filters << :password
+    LogjamAgent.ensure_ping_at_exit = false
+  end
+
+  before '/index' do
+    action_name "Simple#index"
+  end
+
+  get '/index' do
+    logger.info 'Hello World!'
+    'Hello World!'
+  end
+
+  run! if __FILE__ == $0
+end

data/test/sinatra_classic_app.rb

@@ -0,0 +1,31 @@
+$:.unshift File.expand_path('../../lib', __FILE__)
+
+require 'logjam_agent/sinatra'
+
+use LogjamAgent::Sinatra::Middleware
+
+configure do
+  set :root, File.expand_path('../..', __FILE__)
+  set :environment, :test
+  set :loglevel, :debug
+  setup_logjam_logger
+
+  LogjamAgent.application_name = "myapp"
+  LogjamAgent.add_forwarder(
+    :zmq,
+    :host => "localhost",
+    :port => 9604,
+    :linger     => 10,
+    :snd_hwm    => 10,
+    :rcv_hwm    => 10,
+    :rcv_timeo  => 10,
+    :snd_timeo  => 10
+  )
+  LogjamAgent.parameter_filters << :password
+end
+
+get '/index' do
+  action_name "Simple#index"
+  logger.info 'Hello World!'
+  'Hello World!'
+end

data/test/sinatra_classic_test.rb

@@ -0,0 +1,20 @@
+require_relative "test_helper.rb"
+require_relative "sinatra_classic_app"
+require "rack/test"
+
+module LogjamAgent
+  class SinatraClassicTest < MiniTest::Test
+
+    include ::Rack::Test::Methods
+
+    def app
+      ::Sinatra::Application
+    end
+
+    def test_root
+      get '/index?mumu=1&password=5'
+      assert_equal 'Hello World!', last_response.body
+    end
+
+  end
+end

data/test/sinatra_test.rb

@@ -0,0 +1,54 @@
+require_relative "test_helper.rb"
+require_relative "sinatra_app"
+require "rack/test"
+
+module LogjamAgent
+  class SinatraTest < MiniTest::Test
+    def setup
+      @@receiver ||= LogjamAgent::Receiver.new
+      LogjamAgent.enable!
+    end
+
+    def teardown
+      LogjamAgent.disable!
+    end
+
+    def logjam_message
+      @@receiver.receive
+    end
+
+    include ::Rack::Test::Methods
+
+    def app
+      SinatraTestApp
+    end
+
+    def test_root
+      get '/index?mumu=1&password=5'
+      assert_equal 'Hello World!', last_response.body
+      assert_equal 200, last_response.status
+
+      stream, topic, payload = logjam_message
+      assert_equal "myapp-test", stream
+      assert_equal "logs.myapp.test", topic
+      assert_equal 200, payload["code"]
+      assert_equal "Simple#index", payload["action"]
+      assert_kind_of Float, payload["total_time"]
+      assert_kind_of String, payload["started_at"]
+      assert_kind_of Integer, payload["started_ms"]
+      assert_kind_of String, payload["ip"]
+      # assert_kind_of Float, payload["view_time"]
+      lines = payload["lines"]
+      assert_match(/Started GET.*password=\[FILTERED\]/, lines[0][2])
+      assert_match(/Hello World/, lines[1][2])
+      assert_match(/Completed 200 OK/, lines[2][2])
+      assert_nil(lines[3])
+      request_info = payload["request_info"]
+      method, url, query_parameters = request_info.values_at(*%w(method url query_parameters))
+      assert_equal method, "GET"
+      assert_equal url, "/index?mumu=1&password=[FILTERED]"
+      assert_equal(query_parameters, { "mumu" => "1", "password" => "[FILTERED]" })
+    end
+
+  end
+end

data/test/test_helper.rb

@@ -8,4 +8,9 @@ class MiniTest::Test
   extend ActiveSupport::Testing::Declarative
 end
 
-require_relative "../lib/logjam_agent"
+$:.unshift File.expand_path('../../lib', __FILE__)
+require "logjam_agent"
+require "logjam_agent/receiver"
+
+# for Sinatra
+ENV['RACK_ENV'] = "test"

data/test/util_test.rb

@@ -24,5 +24,10 @@ module LogjamAgent
       assert_equal "tcp://a:1", augmented
     end
 
+    test "supports inproc" do
+      augmented = augment_connection_spec("inproc://hugo-schnugo", 2)
+      assert_equal "inproc://hugo-schnugo", augmented
+    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logjam_agent
 version: !ruby/object:Gem::Version
-  version: 0.30.0
+  version: 0.31.0
 platform: ruby
 authors:
 - Stefan Kaes
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-15 00:00:00.000000000 Z
+date: 2020-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -122,6 +122,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +184,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.12.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.12.2
 - !ruby/object:Gem::Dependency
   name: ffi-rzmq-core
   requirement: !ruby/object:Gem::Requirement
@@ -185,32 +227,39 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- Gemfile
 - README.md
 - Rakefile
 - lib/logjam_agent.rb
 - lib/logjam_agent/actionpack/lib/action_dispatch/middleware/remote_ip.rb
+- lib/logjam_agent/active_support/core_ext/array/extract.rb
+- lib/logjam_agent/active_support/parameter_filter.rb
 - lib/logjam_agent/buffered_logger.rb
 - lib/logjam_agent/forwarders.rb
 - lib/logjam_agent/middleware.rb
 - lib/logjam_agent/rack/logger.rb
+- lib/logjam_agent/rack/rails_support.rb
+- lib/logjam_agent/rack/sinatra_request.rb
 - lib/logjam_agent/railtie.rb
+- lib/logjam_agent/receiver.rb
 - lib/logjam_agent/request.rb
+- lib/logjam_agent/sinatra.rb
 - lib/logjam_agent/syslog_like_formatter.rb
 - lib/logjam_agent/util.rb
 - lib/logjam_agent/version.rb
 - lib/logjam_agent/zmq_forwarder.rb
-- logjam_agent.gemspec
-- script/console
 - test/request_test.rb
+- test/sinatra_app.rb
+- test/sinatra_classic_app.rb
+- test/sinatra_classic_test.rb
+- test/sinatra_test.rb
 - test/test_helper.rb
 - test/util_test.rb
 - test/zmq_forwarder_test.rb
-homepage: ''
-licenses: []
+homepage: https://github.com/skaes/logjam_agent
+licenses:
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -225,12 +274,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Logjam client library to be used with logjam
 test_files:
+- test/sinatra_app.rb
+- test/sinatra_classic_test.rb
+- test/sinatra_classic_app.rb
 - test/request_test.rb
-- test/test_helper.rb
-- test/util_test.rb
 - test/zmq_forwarder_test.rb
+- test/util_test.rb
+- test/test_helper.rb
+- test/sinatra_test.rb

data/.gitignore

@@ -1,5 +0,0 @@
-*.gem
-.bundle
-Gemfile.lock
-pkg/*
-.rvmrc

data/Gemfile

@@ -1,4 +0,0 @@
-source "http://rubygems.org"
-
-# Specify your gem's dependencies in logjam_agent.gemspec
-gemspec

data/logjam_agent.gemspec

@@ -1,32 +0,0 @@
-# -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "logjam_agent/version"
-
-Gem::Specification.new do |s|
-  s.name        = "logjam_agent"
-  s.version     = LogjamAgent::VERSION
-  s.authors     = ["Stefan Kaes"]
-  s.email       = ["stefan.kaes@xing.com"]
-  s.homepage    = ""
-  s.summary     = %q{Logjam client library to be used with logjam}
-  s.description = %q{Logjam logger and request information forwarding}
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
-
-  s.add_development_dependency "rake"
-  s.add_development_dependency "i18n"
-  s.add_development_dependency "snappy"
-  s.add_development_dependency "lz4-ruby"
-  s.add_development_dependency "oj"
-  s.add_development_dependency "byebug"
-  s.add_development_dependency "minitest"
-  s.add_development_dependency "mocha"
-
-  s.add_runtime_dependency "activesupport"
-  s.add_runtime_dependency "time_bandits", [">= 0.6.0"]
-  s.add_runtime_dependency "ffi-rzmq-core", [">= 1.0.5"]
-  s.add_runtime_dependency "ffi-rzmq", [">= 2.0.4"]
-end

data/script/console

@@ -1,28 +0,0 @@
-#!/usr/bin/env ruby
-require 'optparse'
-
-options = { }
-OptionParser.new do |opt|
-  opt.banner = "Usage: console [options]"
-  opt.on("--debugger", 'Enable ruby-debugging for the console.') { |v| options[:debugger] = v }
-  opt.parse!(ARGV)
-end
-
-lib_dir = File.expand_path("../../lib/", __FILE__)
-libs =  " -r irb/completion"
-libs << " -I #{lib_dir}"
-libs << " -r #{lib_dir}/logjam_agent.rb"
-
-if options[:debugger]
-  begin
-    require 'byebug'
-    libs << " -r byebug"
-    puts "=> Debugger enabled"
-  rescue Exception
-    puts "You need to install ruby-debug to run the console in debugging mode. With gems, use 'gem install ruby-debug'"
-    exit
-  end
-end
-
-irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
-exec "#{irb} #{libs} --simple-prompt"