oneapm_rpm 1.1.0

data/lib/one_apm/agent/agent/connect.rb

@@ -0,0 +1,175 @@
+module OneApm
+  module Agent
+    class Agent
+      # This module is an artifact of a refactoring of the connect
+      # method - all of its methods are used in that context, so it
+      # can be refactored at will. It should be fully tested
+      module Connect
+        # number of attempts we've made to contact the server
+        attr_accessor :connect_attempts
+
+        # Disconnect just sets connected to false, which prevents
+        # the agent from trying to connect again
+        def disconnect
+          @connect_state = :disconnected
+          true
+        end
+
+        def connected?
+          @connect_state == :connected
+        end
+
+        def disconnected?
+          @connect_state == :disconnected
+        end
+
+        # Don't connect if we're already connected, or if we tried to connect
+        # and were rejected with prejudice because of a license issue, unless
+        # we're forced to by force_reconnect.
+        def should_connect?(force = false)
+          force || (!connected? && !disconnected?)
+        end
+
+        # Retry period is a minute for each failed attempt that
+        # we've made. This should probably do some sort of sane TCP
+        # backoff to prevent hammering the server, but a minute for
+        # each attempt seems to work reasonably well.
+        def connect_retry_period
+          [600, connect_attempts * 60].min
+        end
+
+        def note_connect_failure
+          self.connect_attempts += 1
+        end
+
+        # When we have a problem connecting to the server, we need
+        # to tell the user what happened, since this is not an error
+        # we can handle gracefully.
+        def log_error(error)
+          ::OneApm::Agent.logger.error "Error establishing connection with OneApm Service at #{@service.server}:", error
+        end
+
+        # When the server sends us an error with the license key, we
+        # want to tell the user that something went wrong, and let
+        # them know where to go to get a valid license key
+        #
+        # After this runs, it disconnects the agent so that it will
+        # no longer try to connect to the server, saving the
+        # application and the server load
+        def handle_license_error(error)
+          ::OneApm::Agent.logger.error( \
+            error.message, \
+            "Visit oneapm.com to obtain a valid license key, or to upgrade your account.")
+          disconnect
+        end
+
+        def handle_unrecoverable_agent_error(error)
+          ::OneApm::Agent.logger.error(error.message)
+          disconnect
+          shutdown
+        end
+
+        def generate_environment_report
+          @environment_report = environment_for_connect
+        end
+
+        # Checks whether we should send environment info, and if so,
+        # returns the snapshot from the local environment.
+        # Generating the EnvironmentReport has the potential to trigger
+        # require calls in Rails environments, so this method should only
+        # be called synchronously from on the main thread.
+        def environment_for_connect
+          Agent.config[:send_environment_info] ? Array(EnvironmentReport.new) : []
+        end
+
+        # We've seen objects in the environment report (Rails.env in
+        # particular) that can't seralize to JSON. Cope with that here and
+        # clear out so downstream code doesn't have to check again.
+        def sanitize_environment_report
+          if !@service.valid_to_marshal?(@environment_report)
+            @environment_report = []
+          end
+        end
+
+        # Initializes the hash of settings that we send to the
+        # server. Returns a literal hash containing the options
+        def connect_settings
+          sanitize_environment_report
+          {
+            :pid => $$,
+            :host => local_host,
+            :app_name => Agent.config.app_names,
+            :language => 'ruby',
+            :labels => Agent.config.parsed_labels,
+            :agent_version => OneApm::VERSION::STRING,
+            :environment => @environment_report,
+            :settings => Agent.config.to_collector_hash,
+            :high_security => Agent.config[:high_security],
+            :identifier => "ruby:#{local_host}"
+          }
+        end
+
+        # apdex_f is always 4 times the apdex_t
+        def apdex_f
+          (4 * Agent.config[:apdex_t]).to_f
+        end
+
+        # Sets the collector host and connects to the server, then
+        # invokes the final configuration with the returned data
+        def query_server_for_configuration
+          finish_setup(connect_to_server)
+        end
+
+        # Returns connect data passed back from the server
+        def connect_to_server
+          @service.connect(connect_settings)
+        end
+
+        # Takes a hash of configuration data returned from the
+        # server and uses it to set local variables and to
+        # initialize various parts of the agent that are configured
+        # separately.
+        #
+        # Can accommodate most arbitrary data - anything extra is
+        # ignored unless we say to do something with it here.
+        def finish_setup(config_data)
+          return if config_data == nil
+
+          @service.agent_id = config_data['agent_run_id'] if @service
+
+          ::OneApm::Agent.logger.debug "Server provided config: #{config_data.inspect}"
+          server_config = OneApm::Configuration::ServerSource.new(config_data, Agent.config)
+          Agent.config.replace_or_add_config(server_config)
+          log_connection!(config_data) if @service
+
+          @transaction_rules = OneApm::Support::RulesEngine.create_transaction_rules(config_data)
+          @stats_engine.metric_rules = OneApm::Support::RulesEngine.create_metric_rules(config_data)
+
+          # If you're adding something else here to respond to the server-side config,
+          # use Agent.instance.events.subscribe(:finished_configuring) callback instead!
+        end
+
+        def local_host
+          @local_host ||= OneApm::Agent::Hostname.get
+        end
+
+        # Logs when we connect to the server, for debugging purposes
+        # - makes sure we know if an agent has not connected
+        def log_connection!(config_data)
+          ::OneApm::Agent.logger.debug "Connected to OneApm Service at #{@service.collector.name}"
+          ::OneApm::Agent.logger.debug "Agent Run       = #{@service.agent_id}."
+          ::OneApm::Agent.logger.debug "Connection data = #{config_data.inspect}"
+          if config_data['messages'] && config_data['messages'].any?
+            log_collector_messages(config_data['messages'])
+          end
+        end
+
+        def log_collector_messages(messages)
+          messages.each do |message|
+            ::OneApm::Agent.logger.send(message['level'].downcase, message['message'])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/agent/agent/container_data_manager.rb

@@ -0,0 +1,218 @@
+module OneApm
+  module Agent
+    class Agent
+      module ContainerDataManager
+
+        # the statistics engine that holds all the timeslice data
+        attr_reader :stats_engine
+        # the transaction sampler that handles recording transactions
+        attr_reader :transaction_sampler
+        attr_reader :sql_sampler
+        # manages agent commands we receive from the collector, and the handlers
+        attr_reader :agent_command_router
+        # error collector is a simple collection of recorded errors
+        attr_reader :error_collector
+        attr_reader :custom_event_aggregator
+
+        def merge_data_for_endpoint(endpoint, data)
+          if data && !data.empty?
+            container_for_endpoint(endpoint).merge!(data)
+          end
+        rescue => e
+          OneApm::Agent.logger.error("Error while merging #{endpoint} data from child: ", e)
+        end
+
+        # Clear out the metric data, errors, and transaction traces, etc.
+        def drop_buffered_data
+          @stats_engine.reset!
+          @error_collector.reset!
+          @transaction_sampler.reset!
+          @transaction_event_aggregator.reset!
+          @custom_event_aggregator.reset!
+          @sql_sampler.reset!
+        end
+
+        private
+
+        def init_containers(events)
+          @stats_engine                 = OneApm::Collector::StatsEngine.new
+          @transaction_sampler          = OneApm::Collector::TransactionSampler.new
+          @sql_sampler                  = OneApm::Collector::SqlSampler.new
+          @error_collector              = OneApm::Collector::ErrorCollector.new
+          @custom_event_aggregator      = OneApm::Collector::CustomEventAggregator.new
+          @transaction_event_aggregator = OneApm::Collector::TransactionEventAggregator.new(@events)
+          @utilization_data             = OneApm::Collector::UtilizationData.new
+          @agent_command_router         = OneApm::Collector::AgentCommandRouter.new(@events)
+        end
+
+        def container_for_endpoint(endpoint)
+          case endpoint
+          when :metric_data             then @stats_engine
+          when :transaction_sample_data then @transaction_sampler
+          when :sql_trace_data          then @sql_sampler
+          when :error_data              then @error_collector
+          when :custom_event_data       then @custom_event_aggregator
+          when :analytic_event_data     then @transaction_event_aggregator
+          end
+        end
+
+        # Harvests data from the given container, sends it to the named endpoint
+        # on the service, and automatically merges back in upon a recoverable
+        # failure.
+        #
+        # The given container should respond to:
+        #
+        #  #harvest!
+        #    returns an enumerable collection of data items to be sent to the
+        #    collector.
+        #
+        #  #reset!
+        #    drop any stored data and reset to a clean state.
+        #
+        #  #merge!(items)
+        #    merge the given items back into the internal buffer of the
+        #    container, so that they may be harvested again later.
+        #
+        def harvest_and_send_from_container(container, endpoint)
+          items = harvest_from_container(container, endpoint)
+          send_data_to_endpoint(endpoint, items, container) unless items.empty?
+        end
+
+        def harvest_from_container(container, endpoint)
+          items = []
+          begin
+            items = container.harvest!
+          rescue => e
+            OneApm::Agent.logger.error("Failed to harvest #{endpoint} data, resetting. Error: ", e)
+            container.reset!
+          end
+          items
+        end
+
+        def send_data_to_endpoint(endpoint, items, container)
+          OneApm::Agent.logger.debug("Sending #{items.size} items to #{endpoint}")
+          begin
+            @service.send(endpoint, items)
+          rescue ForceRestartException, ForceDisconnectException
+            raise
+          rescue SerializationError => e
+            OneApm::Agent.logger.warn("Failed to serialize data for #{endpoint}, discarding. Error: ", e)
+          rescue UnrecoverableServerException => e
+            OneApm::Agent.logger.warn("#{endpoint} data was rejected by remote service, discarding. Error: ", e)
+          rescue ServerConnectionException => e
+            log_remote_unavailable(endpoint, e)
+            container.merge!(items)
+          rescue => e
+            OneApm::Agent.logger.info("Unable to send #{endpoint} data, will try again later. Error: ", e)
+            container.merge!(items)
+          end
+        end
+
+        def harvest_and_send_timeslice_data
+          OneApm::Agent::BusyCalculator.harvest_busy
+          harvest_and_send_from_container(@stats_engine, :metric_data)
+        end
+
+        def harvest_and_send_slowest_sql
+          harvest_and_send_from_container(@sql_sampler, :sql_trace_data)
+        end
+
+        # This handles getting the transaction traces and then sending
+        # them across the wire.  This includes gathering SQL
+        # explanations, stripping out stack traces, and normalizing
+        # SQL.  note that we explain only the sql statements whose
+        # segments' execution times exceed our threshold.
+        def harvest_and_send_transaction_traces
+          harvest_and_send_from_container(@transaction_sampler, :transaction_sample_data)
+        end
+
+        def harvest_and_send_for_agent_commands
+          harvest_and_send_from_container(@agent_command_router, :profile_data)
+        end
+
+        def harvest_and_send_errors
+          harvest_and_send_from_container(@error_collector, :error_data)
+        end
+
+        def harvest_and_send_analytic_event_data
+          harvest_and_send_from_container(@transaction_event_aggregator, :analytic_event_data)
+          harvest_and_send_from_container(@custom_event_aggregator,      :custom_event_data)
+        end
+
+        def harvest_and_send_utilization_data
+          harvest_and_send_from_container(@utilization_data, :utilization_data)
+        end
+
+        def transmit_data
+          harvest_lock.synchronize do
+            transmit_data_already_locked
+          end
+        end
+
+        def transmit_event_data
+          transmit_single_data_type(:harvest_and_send_analytic_event_data, "TransactionEvent")
+        end
+
+        def transmit_utilization_data
+          transmit_single_data_type(:harvest_and_send_utilization_data, "UtilizationData")
+        end
+
+        def transmit_single_data_type(harvest_method, supportability_name)
+          now = Time.now
+
+          msg = "Sending #{harvest_method.to_s.gsub("harvest_and_send_", "")} to OneApm Service"
+          ::OneApm::Agent.logger.debug msg
+
+          harvest_lock.synchronize do
+            @service.session do # use http keep-alive
+              self.send(harvest_method)
+            end
+          end
+        ensure
+          duration = (Time.now - now).to_f
+          OneApm::Agent.record_metric("Supportability/#{supportability_name}Harvest", duration)
+        end
+
+        # This method is expected to only be called with the harvest_lock already held
+        def transmit_data_already_locked
+          now = Time.now
+          ::OneApm::Agent.logger.debug "Sending data to OneApm Service"
+
+          @events.notify(:before_harvest)
+          @service.session do
+            harvest_and_send_errors
+            harvest_and_send_transaction_traces
+            harvest_and_send_slowest_sql
+            harvest_and_send_timeslice_data
+
+            check_for_and_handle_agent_commands
+            harvest_and_send_for_agent_commands
+          end
+        ensure
+          OneApm::Agent::Database.close_connections
+          duration = (Time.now - now).to_f
+          OneApm::Agent.record_metric('Supportability/Harvest', duration)
+        end
+
+        def check_for_and_handle_agent_commands
+          begin
+            @agent_command_router.check_for_and_handle_agent_commands
+          rescue ForceRestartException, ForceDisconnectException
+            raise
+          rescue ServerConnectionException => e
+            log_remote_unavailable(:get_agent_commands, e)
+          rescue => e
+            OneApm::Agent.logger.info("Error during check_for_and_handle_agent_commands, will retry later: ", e)
+          end
+        end
+
+        def log_remote_unavailable(endpoint, e)
+          OneApm::Agent.logger.debug("Unable to send #{endpoint} data, will try again later. Error: ", e)
+          OneApm::Agent.record_metric("Supportability/remote_unavailable", 0.0)
+          OneApm::Agent.record_metric("Supportability/remote_unavailable/#{endpoint.to_s}", 0.0)
+        end
+
+      end
+    end
+  end
+end

data/lib/one_apm/agent/agent/forkable_dispatcher_functions.rb

@@ -0,0 +1,96 @@
+module OneApm
+  module Agent
+    class Agent
+      module ForkableDispatcherFunctions
+
+        # This method should be called in a forked process after a fork.
+        # It assumes the parent process initialized the agent, but does
+        # not assume the agent started.
+        #
+        # The call is idempotent, but not re-entrant.
+        #
+        # * It clears any metrics carried over from the parent process
+        # * Restarts the sampler thread if necessary
+        # * Initiates a new agent run and worker loop unless that was done
+        #   in the parent process and +:force_reconnect+ is not true
+        #
+        # Options:
+        # * <tt>:force_reconnect => true</tt> to force the spawned process to
+        #   establish a new connection, such as when forking a long running process.
+        #   The default is false--it will only connect to the server if the parent
+        #   had not connected.
+        # * <tt>:keep_retrying => false</tt> if we try to initiate a new
+        #   connection, this tells me to only try it once so this method returns
+        #   quickly if there is some kind of latency with the server.
+        def after_fork(options={})
+          needs_restart = false
+          @after_fork_lock.synchronize do
+            needs_restart = @harvester.needs_restart?
+            @harvester.mark_started
+          end
+
+          return if !needs_restart ||
+            !Agent.config[:agent_enabled] ||
+            !Agent.config[:monitor_mode] ||
+            disconnected?
+
+          ::OneApm::Agent.logger.debug "Starting the worker thread in #{Process.pid} (parent #{Process.ppid}) after forking."
+
+          channel_id = options[:report_to_channel]
+          install_pipe_service(channel_id) if channel_id
+
+          # Clear out locks and stats left over from parent process
+          reset_objects_with_locks
+          drop_buffered_data
+
+          setup_and_start_agent(options)
+        end
+
+        def install_pipe_service(channel_id)
+          @service = OneApm::Agent::PipeService.new(channel_id)
+          if connected?
+            @connected_pid = Process.pid
+          else
+            ::OneApm::Agent.logger.debug("Child process #{Process.pid} not reporting to non-connected parent (process #{Process.ppid}).")
+            @service.shutdown(Time.now)
+            disconnect
+          end
+        end
+
+        # Synchronize with the harvest loop. If the harvest thread has taken
+        # a lock (DNS lookups, backticks, agent-owned locks, etc), and we
+        # fork while locked, this can deadlock child processes. For more
+        # details, see https://github.com/resque/resque/issues/1101
+        def synchronize_with_harvest
+          harvest_lock.synchronize do
+            yield
+          end
+        end
+
+        # Clear out state for any objects that we know lock from our parents
+        # This is necessary for cases where we're in a forked child and Ruby
+        # might be holding locks for background thread that aren't there anymore.
+        def reset_objects_with_locks
+          @stats_engine = OneApm::Collector::StatsEngine.new
+          reset_harvest_locks
+        end
+
+        # Some forking cases (like Resque) end up with harvest lock from the
+        # parent process orphaned in the child. Let it go before we proceed.
+        def reset_harvest_locks
+          return if harvest_lock.nil?
+
+          harvest_lock.unlock if harvest_lock.locked?
+        end
+
+        def flush_pipe_data
+          if connected? && @service.is_a?(::OneApm::Agent::PipeService)
+            transmit_data
+            transmit_event_data
+          end
+        end
+
+      end
+    end
+  end
+end