mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/agent_logger.rb

@@ -0,0 +1,184 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'thread'
+require 'logger'
+require 'mmtrix/agent/hostname'
+require 'mmtrix/agent/log_once'
+
+module Mmtrix
+  module Agent
+    class AgentLogger
+      include LogOnce
+
+      def initialize(root = "", override_logger=nil)
+        @already_logged_lock = Mutex.new
+        clear_already_logged
+        create_log(root, override_logger)
+        set_log_level!
+        set_log_format!
+
+        gather_startup_logs
+      end
+
+      def fatal(*msgs, &blk)
+        format_and_send(:fatal, msgs, &blk)
+      end
+
+      def error(*msgs, &blk)
+        format_and_send(:error, msgs, &blk)
+      end
+
+      def warn(*msgs, &blk)
+        format_and_send(:warn, msgs, &blk)
+      end
+
+      def info(*msgs, &blk)
+        format_and_send(:info, msgs, &blk)
+      end
+
+      def debug(*msgs, &blk)
+        format_and_send(:debug, msgs, &blk)
+      end
+
+      def is_startup_logger?
+        @log.is_a?(NullLogger)
+      end
+
+      # Use this when you want to log an exception with explicit control over
+      # the log level that the backtrace is logged at. If you just want the
+      # default behavior of backtraces logged at debug, use one of the methods
+      # above and pass an Exception as one of the args.
+      def log_exception(level, e, backtrace_level=level)
+        @log.send(level, "%p: %s" % [ e.class, e.message ])
+        @log.send(backtrace_level) do
+          backtrace = backtrace_from_exception(e)
+          if backtrace
+            "Debugging backtrace:\n" + backtrace.join("\n  ")
+          else
+            "No backtrace available."
+          end
+        end
+      end
+
+      def log_formatter=(formatter)
+        @log.formatter = formatter
+      end
+
+      private
+
+      def backtrace_from_exception(e)
+        # We've seen that often the backtrace on a SystemStackError is bunk
+        # so massage the caller instead at a known depth.
+        #
+        # Tests keep us honest about minmum method depth our log calls add.
+        return caller.drop(5) if e.is_a?(SystemStackError)
+
+        e.backtrace
+      end
+
+      # Allows for passing exceptions in explicitly, which format with backtrace
+      def format_and_send(level, *msgs, &block)
+        if block
+          if @log.send("#{level}?")
+            msgs = Array(block.call)
+          else
+            msgs = []
+          end
+        end
+
+        msgs.flatten.each do |item|
+          case item
+          when Exception then log_exception(level, item, :debug)
+          else @log.send(level, item)
+          end
+        end
+        nil
+      end
+
+      def create_log(root, override_logger)
+        if !override_logger.nil?
+          @log = override_logger
+        elsif ::Mmtrix::Agent.config[:agent_enabled] == false
+          create_null_logger
+        else
+          if wants_stdout?
+            @log = ::Logger.new(STDOUT)
+          else
+            create_log_to_file(root)
+          end
+        end
+      end
+
+      def create_log_to_file(root)
+        path = find_or_create_file_path(::Mmtrix::Agent.config[:log_file_path], root)
+        if path.nil?
+          @log = ::Logger.new(STDOUT)
+          warn("Error creating log directory #{::Mmtrix::Agent.config[:log_file_path]}, using standard out for logging.")
+        else
+          file_path = "#{path}/#{::Mmtrix::Agent.config[:log_file_name]}"
+          begin
+            @log = ::Logger.new(file_path)
+          rescue => e
+            @log = ::Logger.new(STDOUT)
+            warn("Failed creating logger for file #{file_path}, using standard out for logging.", e)
+          end
+        end
+      end
+
+      def create_null_logger
+        @log = ::Mmtrix::Agent::NullLogger.new
+      end
+
+      def wants_stdout?
+        ::Mmtrix::Agent.config[:log_file_path].upcase == "STDOUT"
+      end
+
+      def find_or_create_file_path(path_setting, root)
+        for abs_path in [ File.expand_path(path_setting),
+                          File.expand_path(File.join(root, path_setting)) ] do
+          if File.directory?(abs_path) || (Dir.mkdir(abs_path) rescue nil)
+            return abs_path[%r{^(.*?)/?$}]
+          end
+        end
+        nil
+      end
+
+      def set_log_level!
+        @log.level = AgentLogger.log_level_for(::Mmtrix::Agent.config[:log_level])
+      end
+
+      LOG_LEVELS = {
+        "debug" => ::Logger::DEBUG,
+        "info"  => ::Logger::INFO,
+        "warn"  => ::Logger::WARN,
+        "error" => ::Logger::ERROR,
+        "fatal" => ::Logger::FATAL,
+      }
+
+      def self.log_level_for(level)
+        LOG_LEVELS.fetch(level.to_s.downcase, ::Logger::INFO)
+      end
+
+      def set_log_format!
+        @hostname = Mmtrix::Agent::Hostname.get
+        @prefix = wants_stdout? ? '** [Mmtrix]' : ''
+        @log.formatter = Proc.new do |severity, timestamp, progname, msg|
+          "#{@prefix}[#{timestamp.strftime("%m/%d/%y %H:%M:%S %z")} #{@hostname} (#{$$})] #{severity} : #{msg}\n"
+        end
+      end
+
+      def gather_startup_logs
+        StartupLogger.instance.dump(self)
+      end
+
+    end
+
+    # In an effort to not lose messages during startup, we trap them in memory
+    # The real logger will then dump its contents out when it arrives.
+    class StartupLogger < MemoryLogger
+      include Singleton
+    end
+  end
+end

data/lib/mmtrix/agent/attribute_filter.rb

@@ -0,0 +1,242 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+# This class applies filtering rules as specified in the Agent Attributes
+# cross-agent spec.
+#
+# Instances of it are constructed by deriving a set of rules from the agent
+# configuration. Instances are immutable once they are constructed - if the
+# config changes, a new instance should be constructed and swapped in to
+# replace the existing one.
+#
+# The #apply method is the main external interface of this class. It takes an
+# attribute name and a set of default destinations (represented as a bitfield)
+# and returns a set of actual destinations after applying the filtering rules
+# specified in the config.
+#
+# Each set of destinations is represented as a bitfield, where the bit positions
+# specified in the DST_* constants are used to indicate whether an attribute
+# should be sent to the corresponding destination.
+#
+# The choice of a bitfield here rather than an Array was made to avoid the need
+# for any transient object allocations during the application of rules. Since
+# rule application will happen once per attribute per transaction, this is a hot
+# code path.
+#
+# The algorithm for applying filtering rules is as follows:
+#
+# 1. Start with a bitfield representing the set of default destinations passed
+#    in to #apply.
+# 2. Mask this bitfield against the set of destinations that have attribute
+#    enabled at all.
+# 3. Traverse the list of rules in order (more on the ordering later), applying
+#    each matching rule, but taking care to not let rules override the enabled
+#    status of each destination. Each matching rule may mutate the bitfield.
+# 4. Return the resulting bitfield after all rules have been applied.
+#
+# Each rule consists of a name, a flag indicating whether it ends with a
+# wildcard, a bitfield representing the set of destinations that it applies to,
+# and a flag specifying whether it is an include or exclude rule.
+#
+# During construction, rules are sorted according to the following criteria:
+#
+# 1. First, the names are compared lexicographically. This has the impact of
+#    forcing shorter (more general) rules towards the top of the list and longer
+#    (more specific) rules towards the bottom. This is important, because the
+#    Agent Attributes spec stipulates that the most specific rule for a given
+#    destination should take precedence. Since rules are applied top-to-bottom,
+#    this sorting guarantees that the most specific rule will be applied last.
+# 2. If the names are identical, we next examine the wildcard flag. Rules ending
+#    with a wildcard are considered more general (and thus 'less than') rules
+#    not ending with a wildcard.
+# 3. If the names and wildcard flags are identical, we next examine whether the
+#    rules being compared are include or exclude rules. Exclude rules have
+#    precedence by the spec, so they are considered 'greater than' include
+#    rules.
+#
+# This approach to rule evaluation was taken from the PHP agent's
+# implementation.
+#
+
+module Mmtrix
+  module Agent
+    class AttributeFilter
+      DST_NONE = 0x0
+
+      DST_TRANSACTION_EVENTS = 1 << 0
+      DST_TRANSACTION_TRACER = 1 << 1
+      DST_ERROR_COLLECTOR    = 1 << 2
+      DST_BROWSER_MONITORING = 1 << 3
+      DST_DEVELOPER_MODE     = 1 << 4
+
+      DST_ALL = 0xF
+
+      attr_reader :rules
+
+      def initialize(config)
+        @enabled_destinations = DST_NONE
+
+        @enabled_destinations |= DST_TRANSACTION_TRACER if config[:'transaction_tracer.attributes.enabled']
+        @enabled_destinations |= DST_TRANSACTION_EVENTS if config[:'transaction_events.attributes.enabled']
+        @enabled_destinations |= DST_ERROR_COLLECTOR    if config[:'error_collector.attributes.enabled']
+        @enabled_destinations |= DST_BROWSER_MONITORING if config[:'browser_monitoring.attributes.enabled']
+
+        @enabled_destinations = DST_NONE unless config[:'attributes.enabled']
+
+        @rules = []
+
+        build_rule(config[:'attributes.exclude'], DST_ALL, false)
+        build_rule(config[:'transaction_tracer.attributes.exclude'], DST_TRANSACTION_TRACER, false)
+        build_rule(config[:'transaction_events.attributes.exclude'], DST_TRANSACTION_EVENTS, false)
+        build_rule(config[:'error_collector.attributes.exclude'],    DST_ERROR_COLLECTOR,    false)
+        build_rule(config[:'browser_monitoring.attributes.exclude'], DST_BROWSER_MONITORING, false)
+
+        build_rule(['request.parameters.*'], include_destinations_for_capture_params(config[:capture_params]), true)
+        build_rule(['job.resque.args.*'],    include_destinations_for_capture_params(config[:'resque.capture_params']), true)
+        build_rule(['job.sidekiq.args.*'],   include_destinations_for_capture_params(config[:'sidekiq.capture_params']), true)
+
+        build_rule(config[:'attributes.include'], DST_ALL, true)
+        build_rule(config[:'transaction_tracer.attributes.include'], DST_TRANSACTION_TRACER, true)
+        build_rule(config[:'transaction_events.attributes.include'], DST_TRANSACTION_EVENTS, true)
+        build_rule(config[:'error_collector.attributes.include'],    DST_ERROR_COLLECTOR,    true)
+        build_rule(config[:'browser_monitoring.attributes.include'], DST_BROWSER_MONITORING, true)
+
+        @rules.sort!
+
+        # We're ok to cache high security for fast lookup because the attribute
+        # filter is re-generated on any significant config change.
+        @high_security = config[:high_security]
+
+        cache_prefix_blacklist
+      end
+
+      def include_destinations_for_capture_params(capturing)
+        if capturing
+          DST_TRANSACTION_TRACER | DST_ERROR_COLLECTOR
+        else
+          DST_NONE
+        end
+      end
+
+      def build_rule(attribute_names, destinations, is_include)
+        attribute_names.each do |attribute_name|
+          rule = AttributeFilterRule.new(attribute_name, destinations, is_include)
+          @rules << rule unless rule.empty?
+        end
+      end
+
+      def apply(attribute_name, default_destinations)
+        return DST_NONE if @enabled_destinations == DST_NONE
+
+        destinations   = default_destinations
+        attribute_name = attribute_name.to_s
+
+        @rules.each do |rule|
+          if rule.match?(attribute_name)
+            if rule.is_include
+              destinations |= rule.destinations
+            else
+              destinations &= rule.destinations
+            end
+          end
+        end
+
+        destinations & @enabled_destinations
+      end
+
+      def allows?(allowed_destinations, requested_destination)
+        allowed_destinations & requested_destination == requested_destination
+      end
+
+      def high_security?
+        @high_security
+      end
+
+      # For attribute prefixes where we know the default destinations will
+      # always be DST_NONE, we can statically determine that any attribute
+      # starting with the prefix will not be allowed unless there's an include
+      # rule that might match attributes starting with it.
+      #
+      # This allows us to skip significant preprocessing work (hash/array
+      # flattening and type coercion) for HTTP request parameters and job
+      # arguments for Sidekiq and Resque in the common case, since none of
+      # these attributes are captured by default.
+      #
+      def cache_prefix_blacklist
+        @prefix_blacklist = {}
+        @prefix_blacklist[:'request.parameters'] = true unless might_allow_prefix_uncached?(:'request.parameters')
+        @prefix_blacklist[:'job.sidekiq.args']   = true unless might_allow_prefix_uncached?(:'job.sidekiq.args')
+        @prefix_blacklist[:'job.resque.args']    = true unless might_allow_prefix_uncached?(:'job.resque.args')
+      end
+
+      # Note that the given prefix *must* be a Symbol
+      def might_allow_prefix?(prefix)
+        !@prefix_blacklist.include?(prefix)
+      end
+
+      def might_allow_prefix_uncached?(prefix)
+        prefix = prefix.to_s
+        @rules.any? do |rule|
+          if rule.is_include
+            if rule.wildcard
+              if rule.attribute_name.size > prefix.size
+                rule.attribute_name.start_with?(prefix)
+              else
+                prefix.start_with?(rule.attribute_name)
+              end
+            else
+              rule.attribute_name.start_with?(prefix)
+            end
+          end
+        end
+      end
+    end
+
+    class AttributeFilterRule
+      attr_reader :attribute_name, :destinations, :is_include, :wildcard
+
+      def initialize(attribute_name, destinations, is_include)
+        @attribute_name = attribute_name.sub(/\*$/, "")
+        @wildcard       = attribute_name.end_with?("*")
+        @is_include     = is_include
+        @destinations   = is_include ? destinations : ~destinations
+      end
+
+      # Rules are sorted from least specific to most specific
+      #
+      # All else being the same, wildcards are considered less specific
+      # All else being the same, include rules are less specific than excludes
+      def <=>(other)
+        name_cmp = @attribute_name <=> other.attribute_name
+        return name_cmp unless name_cmp == 0
+
+        if wildcard != other.wildcard
+          return wildcard ? -1 : 1
+        end
+
+        if is_include != other.is_include
+          return is_include ? -1 : 1
+        end
+
+        return 0
+      end
+
+      def match?(name)
+        if wildcard
+          name.start_with?(@attribute_name)
+        else
+          @attribute_name == name
+        end
+      end
+
+      def empty?
+        if is_include
+          @destinations == AttributeFilter::DST_NONE
+        else
+          @destinations == AttributeFilter::DST_ALL
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/attribute_processing.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module AttributeProcessing
+      module_function
+
+      EMPTY_HASH_STRING_LITERAL = "{}".freeze
+      EMPTY_ARRAY_STRING_LITERAL = "[]".freeze
+
+      def flatten_and_coerce(object, prefix = nil, result = {}, &blk)
+        if object.is_a? Hash
+          flatten_and_coerce_hash(object, prefix, result, &blk)
+        elsif object.is_a? Array
+          flatten_and_coerce_array(object, prefix, result, &blk)
+        elsif prefix
+          val = Coerce.scalar(object)
+          if blk
+            blk.call(prefix, val)
+          elsif !val.nil?
+            result[prefix] = val
+          end
+        else
+          Mmtrix::Agent.logger.warn "Unexpected object: #{object.inspect} with nil prefix passed to Mmtrix::Agent::AttributeProcessing.flatten_and_coerce"
+        end
+        result
+      end
+
+      def flatten_and_coerce_hash(hash, prefix, result, &blk)
+        if hash.empty?
+          if blk
+            blk.call(prefix, EMPTY_HASH_STRING_LITERAL)
+          else
+            result[prefix] = EMPTY_HASH_STRING_LITERAL
+          end
+        else
+          hash.each do |key, val|
+            next_prefix = prefix ? "#{prefix}.#{key}" : key.to_s
+            flatten_and_coerce(val, next_prefix, result, &blk)
+          end
+        end
+      end
+
+      def flatten_and_coerce_array(array, prefix, result, &blk)
+        if array.empty?
+          if blk
+            blk.call(prefix, EMPTY_ARRAY_STRING_LITERAL)
+          else
+            result[prefix] = EMPTY_ARRAY_STRING_LITERAL
+          end
+        else
+          array.each_with_index do |val, idx|
+            next_prefix = prefix ? "#{prefix}.#{idx}" : idx.to_s
+            flatten_and_coerce(val, next_prefix, result, &blk)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/audit_logger.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'logger'
+require 'fileutils'
+require 'mmtrix/agent/hostname'
+
+module Mmtrix
+  module Agent
+    class AuditLogger
+      def initialize
+        @enabled = Mmtrix::Agent.config[:'audit_log.enabled']
+        @endpoints = Mmtrix::Agent.config[:'audit_log.endpoints']
+        @encoder = Mmtrix::Agent::MmtrixService::Encoders::Identity
+      end
+
+      attr_writer :enabled
+
+      def enabled?
+        @enabled
+      end
+
+      def setup?
+        !@log.nil?
+      end
+
+      def log_request(uri, data, marshaller)
+        return unless enabled? && allowed_endpoint?(uri)
+
+        setup_logger unless setup?
+        request_body = if marshaller.class.human_readable?
+          marshaller.dump(data, :encoder => @encoder)
+        else
+          marshaller.prepare(data, :encoder => @encoder).inspect
+        end
+        @log.info("REQUEST: #{uri}")
+        @log.info("REQUEST BODY: #{request_body}")
+      rescue StandardError, SystemStackError, SystemCallError => e
+        ::Mmtrix::Agent.logger.warn("Failed writing to audit log", e)
+      rescue Exception => e
+        ::Mmtrix::Agent.logger.warn("Failed writing to audit log with exception. Re-raising in case of interupt.", e)
+        raise
+      end
+
+      def allowed_endpoint?(uri)
+        @endpoints.any? { |endpoint| uri =~ endpoint }
+      end
+
+      def setup_logger
+        if wants_stdout?
+          # Using $stdout global for easier reassignment in testing
+          @log = ::Logger.new($stdout)
+          ::Mmtrix::Agent.logger.info("Audit log enabled to STDOUT")
+        elsif path = ensure_log_path
+          @log = ::Logger.new(path)
+          ::Mmtrix::Agent.logger.info("Audit log enabled at '#{path}'")
+        else
+          @log = Mmtrix::Agent::NullLogger.new
+        end
+
+        @log.formatter = create_log_formatter
+      end
+
+      def ensure_log_path
+        path = File.expand_path(Mmtrix::Agent.config[:'audit_log.path'])
+        log_dir = File.dirname(path)
+
+        begin
+          FileUtils.mkdir_p(log_dir)
+          FileUtils.touch(path)
+        rescue SystemCallError => e
+          msg = "Audit log disabled, failed opening log at '#{path}': #{e}"
+          ::Mmtrix::Agent.logger.warn(msg)
+          path = nil
+        end
+
+        path
+      end
+
+      def wants_stdout?
+        ::Mmtrix::Agent.config[:'audit_log.path'].upcase == "STDOUT"
+      end
+
+      def create_log_formatter
+        @hostname = Mmtrix::Agent::Hostname.get
+        @prefix = wants_stdout? ? '** [Mmtrix]' : ''
+        Proc.new do |severity, time, progname, msg|
+          "#{@prefix}[#{time} #{@hostname} (#{$$})] : #{msg}\n"
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/autostart.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    # A singleton responsible for determining if the agent should start
+    # monitoring.
+    #
+    # If the agent is in a monitored environment (e.g. production) it will
+    # attempt to avoid starting at "inapproriate" times, for example in an IRB
+    # session.  On Heroku, logs typically go to STDOUT so agent logs can spam
+    # the console during interactive sessions.
+    #
+    # It should be possible to override Autostart logic with an explicit
+    # configuration, for example the MMTRIX_AGENT_ENABLED environment variable or
+    # agent_enabled key in mmtrix.yml
+    module Autostart
+      extend self
+
+
+      # The constants, executables (i.e. $0) and rake tasks used can be
+      # configured with the config keys 'autostart.blacklisted_constants',
+      # 'autostart.blacklisted_executables' and
+      # 'autostart.blacklisted_rake_tasks'
+      def agent_should_start?
+          !blacklisted_constants? &&
+          !blacklisted_executables? &&
+          !in_blacklisted_rake_task?
+      end
+
+      def blacklisted_constants?
+        blacklisted?(Mmtrix::Agent.config[:'autostart.blacklisted_constants']) do |name|
+          constant_is_defined?(name)
+        end
+      end
+
+      def blacklisted_executables?
+        blacklisted?(Mmtrix::Agent.config[:'autostart.blacklisted_executables']) do |bin|
+          File.basename($0) == bin
+        end
+      end
+
+      # Lookup whether namespaced constants (e.g. ::Foo::Bar::Baz) are in the
+      # environment.
+      def constant_is_defined?(const_name)
+        !!::Mmtrix::LanguageSupport.constantize(const_name)
+      end
+
+      def blacklisted?(value, &block)
+        value.split(/\s*,\s*/).any?(&block)
+      end
+
+      def in_blacklisted_rake_task?
+        tasks = begin
+                  ::Rake.application.top_level_tasks
+                rescue => e
+            ::Mmtrix::Agent.logger.debug("Not in Rake environment so skipping blacklisted_rake_tasks check: #{e}")
+            []
+          end
+        !(tasks & ::Mmtrix::Agent.config[:'autostart.blacklisted_rake_tasks'].split(/\s*,\s*/)).empty?
+      end
+    end
+  end
+end