logjam_agent 0.30.0 → 0.32.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b351167b038a91fd392f94501012260ee84319fb6035b9bf5e35811d50c885b1
-  data.tar.gz: 838094a8022bf451422bb66219924737c5d32684c7a6194c979554f101b9071e
+  metadata.gz: '001856dce13e72d273c32137e0cbe7d4c004aa22c788e511b9c409065394de77'
+  data.tar.gz: d6b17a001250b7739ce3fe0e5096bbc57fea7f0632eca09859d28af53b94e368
 SHA512:
-  metadata.gz: 78a136746231e26e873fcddf04740c0d748c29823e9db6fc9eabb43e9fd26cb10ec92f59eb687b70854f20a8fe918476287c171b3eca48f5006226c712ef0fb2
-  data.tar.gz: 84f8e23f8fbbc07e319e8cb928945a1c4f3198502d45a8e9a22e8dc4c9f9184f9283811c24ef40587c41d25badb6bfa3eceea7ca43b2468d3b15c1d49cb24393
+  metadata.gz: 29bec994d20a63997d4b43757cc9a88703cd31208c8146bada5aada8d6bfb7de831bb262ad798441ff42423c59d4aa8acdaf8f730295d15fce98eb3da39e173f
+  data.tar.gz: a27dfebcf4c233fe99b603179cf28a0edc39233b9baca0b3d9df5606e4f3629cd40f8bd832229378cf08c21c51b01963b31474299d6bc5af5289dbb96a11a9af

data/README.md

@@ -5,9 +5,14 @@ Client side library for logjam.
 Hooks into Rails, collects log lines, performance metrics, error/exception infomation and Rack
 environment information and sends this data to [Logjam](https://github.com/skaes/logjam_app).
 
+Has experimental support for Sinatra.
+
 Currently only one mechanism is available for data transport:
 ZeroMQ. Support for AMQP has been dropped.
 
+[![Travis](https://travis-ci.org/skaes/logjam_agent.svg?branch=master)](https://travis-ci.org/github/skaes/logjam_agent)
+
+
 ## Usage
 
 For ZeroMQ, add
@@ -110,10 +115,8 @@ end
 
 ### Generating unique request ids
 
-The agent generates unique request ids for all request handled. It
-will use [uuid4r](https://github.com/skaes/uuid4r) if this is
-available in the application. Otherwise it will fall back to use the
-standard `SecureRandom` class shipped with Ruby.
+The agent generates unique request ids for all request handled using standard
+`SecureRandom` class shipped with Ruby.
 
 ### Generating JSON
 
@@ -121,38 +124,56 @@ The agent will try to use the [Oj](https://github.com/ohler55/oj) to
 generate JSON. If this is not available in your application, it will
 fall back to the `to_json` method.
 
-## Troubleshooting
 
-If the agent experiences problems when sending data, it will log information to a file named
-`logjam_agent_error.log` which you can find under `Rails.root/log`.
-If you set the `RAILS_LOG_TO_STDOUT` environment variable, those logs will be available through `stderr`.
+### Sinatra
 
-This behavior is customizable via a module level call back method:
+Supports both classic and modular Sinatra applications. Since Sinatra doesn't have built
+in action names like Rails, you'll have to declare them in your handlers, or in a before
+filter. Example:
 
 ```ruby
-LogjamAgent.error_handler = lambda {|exception| ... }
+require 'logjam_agent/sinatra'
+
+use LogjamAgent::Sinatra::Middleware
+
+class SinatraTestApp < Sinatra::Base
+  register LogjamAgent::Sinatra
+
+  configure do
+    set :loglevel, :debug
+    setup_logjam_logger
+
+    LogjamAgent.application_name = "myapp"
+    LogjamAgent.add_forwarder(:zmq, :host => "my-logjam-broker")
+    LogjamAgent.parameter_filters << :password
+  end
+
+  before '/index' do
+    action_name "Simple#index"
+  end
+
+  get '/index' do
+    logger.info 'Hello World!'
+    'Hello World!'
+  end
+end
 ```
 
-# License
+The environment name is picked up from either the environment variable `LOGJAM_ENV`, or
+Sinatra's environment setting.
 
-The MIT License
+Set the environment variable `APP_LOG_TO_STDOUT` if you want to log to `STDOUT`.
+Otherwise, logs will appear in the subdirectory `log` of your application's root.
 
-Copyright (c) 2013 - 2019 Stefan Kaes
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+## Troubleshooting
+
+If the agent experiences problems when sending data, it will log information to a file named
+`logjam_agent_error.log` which you can find under `Rails.root/log`.
+If you set the `RAILS_LOG_TO_STDOUT` environment variable, those logs will be available through `stderr`.
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+This behavior is customizable via a module level call back method:
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+```ruby
+LogjamAgent.error_handler = lambda {|exception| ... }
+```

data/Rakefile

@@ -7,8 +7,13 @@ Rake::TestTask.new do |t|
   t.libs << "test"
   t.test_files = FileList['test/**/*_test.rb']
   t.verbose = true
+  t.ruby_opts = %w(-W1)
 end
 
 task :default do
   Rake::Task[:test].invoke
 end
+
+task :integration do
+  sh "cd railsapp && rake"
+end

data/lib/logjam_agent.rb

@@ -1,5 +1,6 @@
 require "socket"
 require "time_bandits"
+require "logjam_agent/monkey_patches/ffi-rzmq-patch"
 
 module LogjamAgent
   module RequestHandling
@@ -36,9 +37,7 @@ require "logjam_agent/request"
 require "logjam_agent/buffered_logger"
 require "logjam_agent/syslog_like_formatter"
 
-if defined?(Rails) && Rails::VERSION::STRING >= "3.0"
-  require "logjam_agent/railtie"
-end
+require "logjam_agent/railtie" if defined?(Rails::Railtie)
 
 # monkey patch log levels to include NONE
 require 'logger'
@@ -74,13 +73,13 @@ module LogjamAgent
   mattr_accessor :action_name_proc
   self.action_name_proc = lambda{|name| name}
 
+  mattr_accessor :parameter_filters
+  self.parameter_filters = []
+
   def self.get_hostname
-    n = Socket.gethostname
-    if n.split('.').size > 1
-      n
-    else
-      Socket.gethostbyname(n).first rescue n
-    end
+    name = Socket.gethostname
+    host = name.split('.').first
+    Addrinfo.getaddrinfo(host, nil, nil, :STREAM, nil, Socket::AI_CANONNAME).first.canonname rescue name
   end
 
   mattr_accessor :hostname
@@ -97,6 +96,9 @@ module LogjamAgent
   mattr_accessor :disabled
   self.disabled = false
 
+  mattr_accessor :ensure_ping_at_exit
+  self.ensure_ping_at_exit = true
+
   mattr_accessor :obfuscate_ips
   self.obfuscate_ips = false
 

data/lib/logjam_agent/active_support/core_ext/array/extract.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Array
+  # Removes and returns the elements for which the block returns a true value.
+  # If no block is given, an Enumerator is returned instead.
+  #
+  #   numbers = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+  #   odd_numbers = numbers.extract! { |number| number.odd? } # => [1, 3, 5, 7, 9]
+  #   numbers # => [0, 2, 4, 6, 8]
+  def extract!
+    return to_enum(:extract!) { size } unless block_given?
+
+    extracted_elements = []
+
+    reject! do |element|
+      extracted_elements << element if yield(element)
+    end
+
+    extracted_elements
+  end
+end

data/lib/logjam_agent/active_support/parameter_filter.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/object/duplicable"
+require_relative "core_ext/array/extract"
+
+module ActiveSupport
+  # +ParameterFilter+ allows you to specify keys for sensitive data from
+  # hash-like object and replace corresponding value. Filtering only certain
+  # sub-keys from a hash is possible by using the dot notation:
+  # 'credit_card.number'. If a proc is given, each key and value of a hash and
+  # all sub-hashes are passed to it, where the value or the key can be replaced
+  # using String#replace or similar methods.
+  #
+  #   ActiveSupport::ParameterFilter.new([:password])
+  #   => replaces the value to all keys matching /password/i with "[FILTERED]"
+  #
+  #   ActiveSupport::ParameterFilter.new([:foo, "bar"])
+  #   => replaces the value to all keys matching /foo|bar/i with "[FILTERED]"
+  #
+  #   ActiveSupport::ParameterFilter.new(["credit_card.code"])
+  #   => replaces { credit_card: {code: "xxxx"} } with "[FILTERED]", does not
+  #   change { file: { code: "xxxx"} }
+  #
+  #   ActiveSupport::ParameterFilter.new([-> (k, v) do
+  #     v.reverse! if k =~ /secret/i
+  #   end])
+  #   => reverses the value to all keys matching /secret/i
+  class ParameterFilter
+    FILTERED = "[FILTERED]" # :nodoc:
+
+    # Create instance with given filters. Supported type of filters are +String+, +Regexp+, and +Proc+.
+    # Other types of filters are treated as +String+ using +to_s+.
+    # For +Proc+ filters, key, value, and optional original hash is passed to block arguments.
+    #
+    # ==== Options
+    #
+    # * <tt>:mask</tt> - A replaced object when filtered. Defaults to +"[FILTERED]"+
+    def initialize(filters = [], mask: FILTERED)
+      @filters = filters
+      @mask = mask
+    end
+
+    # Mask value of +params+ if key matches one of filters.
+    def filter(params)
+      compiled_filter.call(params)
+    end
+
+    # Returns filtered value for given key. For +Proc+ filters, third block argument is not populated.
+    def filter_param(key, value)
+      @filters.empty? ? value : compiled_filter.value_for_key(key, value)
+    end
+
+  private
+    def compiled_filter
+      @compiled_filter ||= CompiledFilter.compile(@filters, mask: @mask)
+    end
+
+    class CompiledFilter # :nodoc:
+      def self.compile(filters, mask:)
+        return lambda { |params| params.dup } if filters.empty?
+
+        strings, regexps, blocks = [], [], []
+
+        filters.each do |item|
+          case item
+          when Proc
+            blocks << item
+          when Regexp
+            regexps << item
+          else
+            strings << Regexp.escape(item.to_s)
+          end
+        end
+
+        deep_regexps = regexps.extract! { |r| r.to_s.include?("\\.") }
+        deep_strings = strings.extract! { |s| s.include?("\\.") }
+
+        regexps << Regexp.new(strings.join("|"), true) unless strings.empty?
+        deep_regexps << Regexp.new(deep_strings.join("|"), true) unless deep_strings.empty?
+
+        new regexps, deep_regexps, blocks, mask: mask
+      end
+
+      attr_reader :regexps, :deep_regexps, :blocks
+
+      def initialize(regexps, deep_regexps, blocks, mask:)
+        @regexps = regexps
+        @deep_regexps = deep_regexps.any? ? deep_regexps : nil
+        @blocks = blocks
+        @mask = mask
+      end
+
+      def call(params, parents = [], original_params = params)
+        filtered_params = params.class.new
+
+        params.each do |key, value|
+          filtered_params[key] = value_for_key(key, value, parents, original_params)
+        end
+
+        filtered_params
+      end
+
+      def value_for_key(key, value, parents = [], original_params = nil)
+        parents.push(key) if deep_regexps
+        if regexps.any? { |r| r.match?(key.to_s) }
+          value = @mask
+        elsif deep_regexps && (joined = parents.join(".")) && deep_regexps.any? { |r| r.match?(joined) }
+          value = @mask
+        elsif value.is_a?(Hash)
+          value = call(value, parents, original_params)
+        elsif value.is_a?(Array)
+          # If we don't pop the current parent it will be duplicated as we
+          # process each array value.
+          parents.pop if deep_regexps
+          value = value.map { |v| value_for_key(key, v, parents, original_params) }
+          # Restore the parent stack after processing the array.
+          parents.push(key) if deep_regexps
+        elsif blocks.any?
+          key = key.dup if key.duplicable?
+          value = value.dup if value.duplicable?
+          blocks.each { |b| b.arity == 2 ? b.call(key, value) : b.call(key, value, original_params) }
+        end
+        parents.pop if deep_regexps
+        value
+      end
+    end
+  end
+end

data/lib/logjam_agent/buffered_logger.rb

@@ -1,54 +1,31 @@
 require 'fileutils'
 
-if ActiveSupport::VERSION::STRING < "4.0"
-  require 'active_support/buffered_logger'
-  require 'active_support/core_ext/logger'
-  if ActiveSupport::VERSION::STRING >= "3.2"
-    require 'active_support/tagged_logging'
-    # monkey patch to handle exceptions correctly
-    # not needed for rails 4 as this uses a Formatter to add the tags
-    class ActiveSupport::TaggedLogging
-      def initialize(logger)
-        @logger = logger
-        if logger.is_a?(LogjamAgent::BufferedLogger)
-          self.class.class_eval <<-EVAL, __FILE__, __LINE__ + 1
-            def add(severity, message = nil, progname = nil, &block)
-              @logger.add(severity, message, progname, tags_text, &block)
-            end
-          EVAL
-        end
-      end
-    end
-  end
-else
-  require 'active_support/logger'
+require 'active_support/logger'
 
-  class LogjamAgent::ConsoleFormatter < Logger::Formatter
-    # This method is invoked when a log event occurs
-    def call(severity, timestamp, progname, msg)
-      "[#{format_time(timestamp)}] #{String === msg ? msg : msg.inspect}\n"
-    end
+class LogjamAgent::ConsoleFormatter < Logger::Formatter
+  # This method is invoked when a log event occurs
+  def call(severity, timestamp, progname, msg)
+    "[#{format_time(timestamp)}] #{String === msg ? msg : msg.inspect}\n"
+  end
 
-    def format_time(timestamp)
-      timestamp.strftime("%H:%M:%S.#{"%06d" % timestamp.usec}")
-    end
+  def format_time(timestamp)
+    timestamp.strftime("%H:%M:%S.#{"%06d" % timestamp.usec}")
   end
+end
 
-  class ActiveSupport::Logger
-    class << self
-      alias_method :original_broadcast, :broadcast
-      def broadcast(logger)
-        logger.formatter = LogjamAgent::ConsoleFormatter.new
-        logger.formatter.extend(ActiveSupport::TaggedLogging::Formatter)
-        original_broadcast(logger)
-      end
+class ActiveSupport::Logger
+  class << self
+    alias_method :original_broadcast, :broadcast
+    def broadcast(logger)
+      logger.formatter = LogjamAgent::ConsoleFormatter.new
+      logger.formatter.extend(ActiveSupport::TaggedLogging::Formatter)
+      original_broadcast(logger)
     end
   end
 end
 
 module LogjamAgent
-  class BufferedLogger < ( ActiveSupport::VERSION::STRING < "4.0" ?
-                           ActiveSupport::BufferedLogger : ActiveSupport::Logger )
+  class BufferedLogger < ActiveSupport::Logger
 
     attr_accessor :formatter
 
@@ -57,8 +34,6 @@ module LogjamAgent
 
     def initialize(*args)
       super(*args)
-      # stupid bug in the buffered logger code (Rails::VERSION::STRING < "3.2")
-      @log.write "\n" if @log && respond_to?(:buffer)
       @formatter = lambda{|_, _, _, message| message}
     end
 

data/lib/logjam_agent/middleware.rb

@@ -1,11 +1,16 @@
 module LogjamAgent
   class Middleware
-    def initialize(app, options={})
+    def initialize(app, framework = :rails)
       @app = app
-      @options = options
+      @framework = framework
+      unless %i{rails sinatra}.include?(framework)
+        raise ArgumentError.new("Invalid logjam_agent framework: #{framework}. Only :rails and :sinatra are valid!")
+      end
+      @reraise = defined?(Rails::Railtie) && Rails.env.test?
     end
 
     def call(env)
+      env["logjam_agent.framework"] = @framework
       strip_encoding_from_etag(env)
       request = start_request(env)
       result = @app.call(env)
@@ -13,9 +18,13 @@ module LogjamAgent
       result
     rescue Exception
       result = [500, {'Content-Type' => 'text/html'}, ["<html><body><h1>500 Internal Server Error</h1></body></html>"]]
+      raise if @reraise
     ensure
       headers = result[1]
       headers["X-Logjam-Request-Id"] = request.id
+      if env["sinatra.static_file"]
+        request.fields[:action] = "Sinatra#static_file"
+      end
       unless (request_action = request.fields[:action]).blank?
         headers["X-Logjam-Request-Action"] = request_action
       end
@@ -41,7 +50,10 @@ module LogjamAgent
       env_name      = env["logjam_agent.environment_name"] || LogjamAgent.environment_name
       caller_id     = env["HTTP_X_LOGJAM_CALLER_ID"] || ""
       caller_action = env["HTTP_X_LOGJAM_ACTION"] || ""
-      LogjamAgent.start_request(app_name, env_name, :caller_id => caller_id, :caller_action => caller_action)
+      extra_fields = {}
+      extra_fields[:caller_id] = caller_id if caller_id.present?
+      extra_fields[:caller_action] = caller_action if caller_action.present?
+      LogjamAgent.start_request(app_name, env_name, extra_fields)
     end
 
     def finish_request(env)