oneapm_rpm 1.1.0

data/lib/one_apm/collector/containers/agent_command_router.rb

@@ -0,0 +1,153 @@
+# encoding: utf-8
+
+require 'one_apm/collector/commands/agent_command'
+require 'one_apm/collector/commands/xray_session_collection'
+require 'one_apm/collector/commands/thread_profiler_session'
+require 'one_apm/agent/threading/backtrace_service'
+
+module OneApm
+  module Collector
+    class AgentCommandRouter
+      attr_reader :handlers
+
+      attr_accessor :thread_profiler_session,
+                    :backtrace_service,
+                    :xray_session_collection
+
+      def initialize(event_listener=nil)
+        @handlers    = Hash.new { |*| Proc.new { |cmd| self.unrecognized_agent_command(cmd) } }
+
+        @backtrace_service = OneApm::Agent::Threading::BacktraceService.new(event_listener)
+
+        @thread_profiler_session = OneApm::Collector::Commands::ThreadProfilerSession.new(@backtrace_service)
+        @xray_session_collection = OneApm::Collector::Commands::XraySessionCollection.new(@backtrace_service, event_listener)
+
+        @handlers['restart'] = Proc.new { OneApm::Agent.restart }
+        @handlers['start_profiler'] = Proc.new { |cmd| thread_profiler_session.handle_start_command(cmd) }
+        @handlers['stop_profiler']  = Proc.new { |cmd| thread_profiler_session.handle_stop_command(cmd) }
+        @handlers['active_xray_sessions'] = Proc.new { |cmd| xray_session_collection.handle_active_xray_sessions(cmd) }
+
+        if event_listener
+          event_listener.subscribe(:before_shutdown, &method(:on_before_shutdown))
+        end
+      end
+
+      def one_apm_service
+        OneApm::Agent.instance.service
+      end
+
+      def check_for_and_handle_agent_commands
+        commands = get_agent_commands
+
+        stop_xray_sessions unless active_xray_command?(commands)
+
+        results = invoke_commands(commands)
+        one_apm_service.agent_command_results(results) unless results.empty?
+      end
+
+      def stop_xray_sessions
+        self.xray_session_collection.stop_all_sessions
+      end
+
+      def active_xray_command?(commands)
+        commands.any? {|command| command.name == 'active_xray_sessions'}
+      end
+
+      def on_before_shutdown(*args)
+        if self.thread_profiler_session.running?
+          self.thread_profiler_session.stop(true)
+        end
+      end
+
+      def harvest!
+        profiles = []
+        profiles += harvest_from_xray_session_collection
+        profiles += harvest_from_thread_profiler_session
+        log_profiles(profiles)
+        profiles
+      end
+
+      # We don't currently support merging thread profiles that failed to send
+      # back into the AgentCommandRouter, so we just no-op this method.
+      # Same with reset! - we don't support asynchronous cancellation of a
+      # running thread profile or X-Ray session currently.
+      def merge!(*args); end
+      def reset!; end
+
+      def harvest_from_xray_session_collection
+        self.xray_session_collection.harvest_thread_profiles
+      end
+
+      def harvest_from_thread_profiler_session
+        if self.thread_profiler_session.ready_to_harvest?
+          self.thread_profiler_session.stop(true)
+          [self.thread_profiler_session.harvest]
+        else
+          []
+        end
+      end
+
+      def log_profiles(profiles)
+        if profiles.empty?
+          ::OneApm::Agent.logger.debug "No thread profiles with data found to send."
+        else
+          profile_descriptions = profiles.map { |p| p.to_log_description }
+          ::OneApm::Agent.logger.debug "Sending thread profiles [#{profile_descriptions.join(", ")}]"
+        end
+      end
+
+      def get_agent_commands
+        commands = one_apm_service.get_agent_commands
+        OneApm::Agent.logger.info "Received get_agent_commands = #{commands.inspect}" if commands.any?
+        commands.map {|collector_command| OneApm::Collector::Commands::AgentCommand.new(collector_command)}
+      end
+
+      def invoke_commands(agent_commands)
+        results = {}
+
+        agent_commands.each do |agent_command|
+          results[agent_command.id.to_s] = invoke_command(agent_command)
+        end
+
+        results
+      end
+
+      class AgentCommandError < StandardError
+      end
+
+      def invoke_command(agent_command)
+        begin
+          call_handler_for(agent_command)
+          return success
+        rescue AgentCommandError => e
+          OneApm::Agent.logger.debug(e)
+          error(e)
+        end
+      end
+
+      SUCCESS_RESULT = {}.freeze
+      ERROR_KEY = "error"
+
+      def success
+        SUCCESS_RESULT
+      end
+
+      def error(err)
+        { ERROR_KEY => err.message }
+      end
+
+      def call_handler_for(agent_command)
+        handler = select_handler(agent_command)
+        handler.call(agent_command)
+      end
+
+      def select_handler(agent_command)
+        @handlers[agent_command.name]
+      end
+
+      def unrecognized_agent_command(agent_command)
+        OneApm::Agent.logger.debug("Unrecognized agent command #{agent_command.inspect}")
+      end
+    end
+  end
+end

data/lib/one_apm/collector/containers/custom_event_aggregator.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+require 'one_apm/support/event_buffer/sized_buffer'
+
+module OneApm
+  module Collector
+    class CustomEventAggregator
+      include OneApm::Coerce
+
+      TYPE             = 'type'.freeze
+      TIMESTAMP        = 'timestamp'.freeze
+      EVENT_PARAMS_CTX = 'recording custom event'.freeze
+      EVENT_TYPE_REGEX = /^[a-zA-Z0-9:_ ]+$/.freeze
+
+      DEFAULT_CAPACITY_KEY = :'custom_insights_events.max_samples_stored'
+
+      def initialize
+        @lock         = Mutex.new
+        @buffer       = OneApm::Agent::SampledBuffer.new(OneApm::Agent.config[DEFAULT_CAPACITY_KEY])
+        @type_strings = Hash.new { |hash, key| hash[key] = key.to_s.freeze }
+        register_config_callbacks
+      end
+
+      def register_config_callbacks
+        OneApm::Agent.config.register_callback(DEFAULT_CAPACITY_KEY) do |max_samples|
+          OneApm::Agent.logger.debug "CustomEventAggregator max_samples set to #{max_samples}"
+          @lock.synchronize do
+            @buffer.capacity = max_samples
+          end
+        end
+      end
+
+      def record(type, attributes)
+        type = @type_strings[type]
+        unless type =~ EVENT_TYPE_REGEX
+          note_dropped_event(type)
+          return false
+        end
+
+        event = [
+          { TYPE => type, TIMESTAMP => Time.now.to_i },
+          event_params(attributes, EVENT_PARAMS_CTX)
+        ]
+
+        stored = @lock.synchronize do
+          @buffer.append(event)
+        end
+        stored
+      end
+
+      def harvest!
+        results = []
+        drop_count = 0
+        @lock.synchronize do
+          results.concat(@buffer.to_a)
+          drop_count += @buffer.num_dropped
+          @buffer.reset!
+        end
+        note_dropped_events(results.size, drop_count)
+        results
+      end
+
+      def note_dropped_events(captured_count, dropped_count)
+        total_count = captured_count + dropped_count
+        if dropped_count > 0
+          OneApm::Agent.logger.warn("Dropped #{dropped_count} custom events out of #{total_count}.")
+        end
+        engine = OneApm::Agent.instance.stats_engine
+        engine.tl_record_supportability_metric_count("Events/Customer/Seen"   ,    total_count)
+        engine.tl_record_supportability_metric_count("Events/Customer/Sent"   , captured_count)
+        engine.tl_record_supportability_metric_count("Events/Customer/Dropped",  dropped_count)
+      end
+
+      def merge!(events)
+        @lock.synchronize do
+          events.each do |event|
+            @buffer.append(event)
+          end
+        end
+      end
+
+      def reset!
+        @lock.synchronize { @buffer.reset! }
+      end
+
+      def note_dropped_event(type)
+        ::OneApm::Agent.logger.log_once(:warn, "dropping_event_of_type:#{type}",
+          "Invalid event type name '#{type}', not recording.")
+        @buffer.note_dropped
+      end
+
+    end
+  end
+end

data/lib/one_apm/collector/containers/error_collector.rb

@@ -0,0 +1,349 @@
+# encoding: utf-8
+
+module OneApm
+  module Collector
+    class ErrorCollector
+      include OneApm::CollectionHelper
+
+      # Defined the methods that need to be stubbed out when the agent is disabled
+      module Shim
+        def notice_error(*args); end
+      end
+
+      # Maximum possible length of the queue - defaults to 20, may be
+      # made configurable in the future. This is a tradeoff between
+      # memory and data retention
+      MAX_ERROR_QUEUE_LENGTH = 20 unless defined? MAX_ERROR_QUEUE_LENGTH
+
+      attr_accessor :errors
+
+      # Returns a new error collector
+      def initialize
+        @errors = []
+
+        # lookup of exception class names to ignore.  Hash for fast access
+        @ignore = {}
+
+        initialize_ignored_errors(Agent.config[:'error_collector.ignore_errors'])
+        @lock = Mutex.new
+
+        Agent.config.register_callback(:'error_collector.enabled') do |config_enabled|
+          ::OneApm::Agent.logger.debug "Errors will #{config_enabled ? '' : 'not '}be sent to the OneApm service."
+        end
+        Agent.config.register_callback(:'error_collector.ignore_errors') do |ignore_errors|
+          initialize_ignored_errors(ignore_errors)
+        end
+      end
+
+      def initialize_ignored_errors(ignore_errors)
+        @ignore.clear
+        ignore_errors = ignore_errors.split(",") if ignore_errors.is_a? String
+        ignore_errors.each { |error| error.strip! }
+        ignore(ignore_errors)
+      end
+
+      def enabled?
+        Agent.config[:'error_collector.enabled']
+      end
+
+      def disabled?
+        !enabled?
+      end
+
+      # We store the passed block in both an ivar on the class, and implicitly
+      # within the body of the ignore_filter_proc method intentionally here.
+      # The define_method trick is needed to get around the fact that users may
+      # call 'return' from within their filter blocks, which would otherwise
+      # result in a LocalJumpError.
+      #
+      # The raw block is also stored in an instance variable so that we can
+      # return it later in its original form.
+      #
+      # This is all done at the class level in order to avoid the case where
+      # the user sets up an ignore filter on one instance of the ErrorCollector,
+      # and then that instance subsequently gets discarded during agent startup.
+      # (For example, if the agent is initially disabled, and then gets enabled
+      # via a call to start later on.)
+      #
+      def self.ignore_error_filter=(block)
+        @ignore_filter = block
+        if block
+          define_method(:ignore_filter_proc, &block)
+        elsif method_defined?(:ignore_filter_proc)
+          undef :ignore_filter_proc
+        end
+        @ignore_filter
+      end
+
+      def self.ignore_error_filter
+        @ignore_filter
+      end
+
+      # errors is an array of Exception Class Names
+      def ignore(errors)
+        errors.each do |error|
+          @ignore[error] = true
+          ::OneApm::Agent.logger.debug("Ignoring errors of type '#{error}'")
+        end
+      end
+
+      # This module was extracted from the notice_error method - it is
+      # internally tested and can be refactored without major issues.
+      module NoticeError
+        # Checks the provided error against the error filter, if there
+        # is an error filter
+        def filtered_by_error_filter?(error)
+          respond_to?(:ignore_filter_proc) && !ignore_filter_proc(error)
+        end
+
+        # Checks the array of error names and the error filter against
+        # the provided error
+        def filtered_error?(error)
+          @ignore[error.class.name] || filtered_by_error_filter?(error)
+        end
+
+        # an error is ignored if it is nil or if it is filtered
+        def error_is_ignored?(error)
+          error && filtered_error?(error)
+        rescue => e
+          OneApm::Agent.logger.error("Error '#{error}' will NOT be ignored. Exception '#{e}' while determining whether to ignore or not.", e)
+          false
+        end
+
+        def seen?(txn, exception)
+          error_ids = txn.nil? ? [] : txn.noticed_error_ids
+          error_ids.include?(exception.object_id)
+        end
+
+        def tag_as_seen(state, exception)
+          txn = state.current_transaction
+          txn.noticed_error_ids << exception.object_id if txn
+        end
+
+        def blamed_metric_name(txn, options)
+          if options[:metric] && options[:metric] != OneApm::Transaction::UNKNOWN_METRIC
+            "Errors/#{options[:metric]}"
+          else
+            "Errors/#{txn.best_name}" if txn
+          end
+        end
+
+        def aggregated_metric_names(txn)
+          metric_names = ["Errors/all"]
+          return metric_names unless txn
+
+          if txn.recording_web_transaction?
+            metric_names << "Errors/allWeb"
+          else
+            metric_names << "Errors/allOther"
+          end
+
+          metric_names
+        end
+
+        # Increments a statistic that tracks total error rate
+        def increment_error_count!(state, exception, options={})
+          txn = state.current_transaction
+
+          metric_names  = aggregated_metric_names(txn)
+          blamed_metric = blamed_metric_name(txn, options)
+          metric_names << blamed_metric if blamed_metric
+
+          stats_engine = OneApm::Agent.instance.stats_engine
+          stats_engine.record_unscoped_metrics(state, metric_names) do |stats|
+            stats.increment_count
+          end
+        end
+
+        def skip_notice_error?(state, exception)
+          disabled? ||
+            error_is_ignored?(exception) ||
+            exception.nil? ||
+            seen?(state.current_transaction, exception)
+        end
+
+        # acts just like Hash#fetch, but deletes the key from the hash
+        def fetch_from_options(options, key, default=nil)
+          options.delete(key) || default
+        end
+
+        # returns some basic option defaults pulled from the provided
+        # options hash
+        def uri_ref_and_root(options)
+          {
+            :request_uri => fetch_from_options(options, :uri, ''),
+            :request_referer => fetch_from_options(options, :referer, ''),
+            :rails_root => OneApm::Probe.instance.root
+          }
+        end
+
+        # If anything else is left over, we treat it like a custom param
+        def custom_params_from_opts(options)
+          # If anything else is left over, treat it like a custom param:
+          if Agent.config[:'error_collector.capture_attributes']
+            fetch_from_options(options, :custom_params, {}).merge(options)
+          else
+            {}
+          end
+        end
+
+        # takes the request parameters out of the options hash, and
+        # returns them if we are capturing parameters, otherwise
+        # returns nil
+        def request_params_from_opts(options)
+          value = options.delete(:request_params)
+          if Agent.config[:capture_params]
+            value
+          else
+            nil
+          end
+        end
+
+        # normalizes the request and custom parameters before attaching
+        # them to the error. See OneApm::CollectionHelper#normalize_params
+        def normalized_request_and_custom_params(options)
+          {
+            :request_params => normalize_params(request_params_from_opts(options)),
+            :custom_params  => normalize_params(custom_params_from_opts(options))
+          }
+        end
+
+        # Merges together many of the options into something that can
+        # actually be attached to the error
+        def error_params_from_options(options)
+          uri_ref_and_root(options).merge(normalized_request_and_custom_params(options))
+        end
+
+        # calls a method on an object, if it responds to it - used for
+        # detection and soft fail-safe. Returns nil if the method does
+        # not exist
+        def sense_method(object, method)
+          object.send(method) if object.respond_to?(method)
+        end
+
+        # extracts a stack trace from the exception for debugging purposes
+        def extract_stack_trace(exception)
+          actual_exception = sense_method(exception, 'original_exception') || exception
+          if bt = sense_method(actual_exception, 'backtrace')
+            bt.reject! { |t| t.include?('one_apm') }
+            bt
+          else
+            '<no stack trace>'
+          end
+        end
+
+        # extracts a bunch of information from the exception to include
+        # in the noticed error - some may or may not be available, but
+        # we try to include all of it
+        def exception_info(exception)
+          {
+            :file_name => sense_method(exception, 'file_name'),
+            :line_number => sense_method(exception, 'line_number'),
+            :stack_trace => extract_stack_trace(exception)
+          }
+        end
+
+        # checks the size of the error queue to make sure we are under
+        # the maximum limit, and logs a warning if we are over the limit.
+        def over_queue_limit?(message)
+          over_limit = (@errors.reject{|err| err.is_internal}.length >= MAX_ERROR_QUEUE_LENGTH)
+          ::OneApm::Agent.logger.warn("The error reporting queue has reached #{MAX_ERROR_QUEUE_LENGTH}. The error detail for this and subsequent errors will not be transmitted to OneApm until the queued errors have been sent: #{message}") if over_limit
+          over_limit
+        end
+
+        # Synchronizes adding an error to the error queue, and checks if
+        # the error queue is too long - if so, we drop the error on the
+        # floor after logging a warning.
+        def add_to_error_queue(noticed_error)
+          @lock.synchronize do
+            if !over_queue_limit?(noticed_error.message) && !@errors.include?(noticed_error)
+              @errors << noticed_error
+            end
+          end
+        end
+      end
+
+      include NoticeError
+
+
+      # Notice the error with the given available options:
+      #
+      # * <tt>:uri</tt> => The request path, minus any request params or query string.
+      # * <tt>:referer</tt> => The URI of the referer
+      # * <tt>:metric</tt> => The metric name associated with the transaction
+      # * <tt>:request_params</tt> => Request parameters, already filtered if necessary
+      # * <tt>:custom_params</tt> => Custom parameters
+      #
+      # If anything is left over, it's added to custom params
+      # If exception is nil, the error count is bumped and no traced error is recorded
+      def notice_error(exception, options={}) #THREAD_LOCAL_ACCESS
+        state = ::OneApm::TransactionState.tl_get
+
+        return if skip_notice_error?(state, exception)
+        tag_as_seen(state, exception)
+
+        increment_error_count!(state, exception, options)
+        OneApm::Agent.instance.events.notify(:notice_error, exception, options)
+
+        action_path       = fetch_from_options(options, :metric, "")
+        exception_options = error_params_from_options(options).merge(exception_info(exception))
+        add_to_error_queue(OneApm::NoticedError.new(action_path, exception_options, exception))
+
+        exception
+      rescue => e
+        ::OneApm::Agent.logger.warn("Failure when capturing error '#{exception}':", e)
+      end
+
+      # *Use sparingly for difficult to track bugs.*
+      #
+      # Track internal agent errors for communication back to OneApm.
+      # To use, make a specific subclass of OneApm::Agent::InternalAgentError,
+      # then pass an instance of it to this method when your problem occurs.
+      #
+      # Limits are treated differently for these errors. We only gather one per
+      # class per harvest, disregarding (and not impacting) the app error queue
+      # limit.
+      def notice_agent_error(exception)
+        return unless exception.class < OneApm::Agent::InternalAgentError
+
+        # Log 'em all!
+        OneApm::Agent.logger.info(exception)
+
+        @lock.synchronize do
+          # Already seen this class once? Bail!
+          return if @errors.any? { |err| err.exception_class_name == exception.class.name }
+
+          trace = exception.backtrace || caller.dup
+          noticed_error = OneApm::NoticedError.new("OneApm/AgentError",
+                                                   {:stack_trace => trace},
+                                                   exception)
+          @errors << noticed_error
+        end
+      rescue => e
+        OneApm::Agent.logger.info("Unable to capture internal agent error due to an exception:", e)
+      end
+
+      def merge!(errors)
+        errors.each do |error|
+          add_to_error_queue(error)
+        end
+      end
+
+      # Get the errors currently queued up.  Unsent errors are left
+      # over from a previous unsuccessful attempt to send them to the server.
+      def harvest!
+        @lock.synchronize do
+          errors = @errors
+          @errors = []
+          errors
+        end
+      end
+
+      def reset!
+        @lock.synchronize do
+          @errors = []
+        end
+      end
+    end
+  end
+end