tingyun_rpm 1.0.6

data/lib/ting_yun/agent/event/event_loop.rb

@@ -0,0 +1,194 @@
+# encoding: utf-8
+# This file is distributed under Ting Yun's license terms.
+require 'ting_yun/support/exception'
+
+
+module TingYun
+  module Agent
+    module Event
+      class EventLoop
+        class Timer
+          attr_reader :next_fire_time, :event, :interval, :last_fired_at
+
+          def initialize(interval, event, repeat=false)
+            @interval      = interval
+            @event         = event
+            @repeat        = repeat
+            @started_at    = Time.now
+            @last_fired_at = nil
+            reschedule
+          end
+
+          def reschedule
+            @next_fire_time = calculate_next_fire_time
+          end
+
+          def advance(amount)
+            @next_fire_time -= amount
+          end
+
+          def last_interval_start
+            @last_fired_at || @started_at
+          end
+
+          def calculate_next_fire_time
+            now = Time.now
+            return now if @interval == 0
+            fire_time = @last_fired_at || now
+            while fire_time <= now
+              fire_time += @interval
+            end
+            fire_time
+          end
+
+          def set_fired_time
+            @last_fired_at = Time.now
+          end
+
+          def due?(now=Time.now)
+            now >= @next_fire_time
+          end
+
+          def finished?
+            !@repeat && @last_fired_at
+          end
+        end
+
+        def initialize
+          @self_pipe_rd, @self_pipe_wr = IO.pipe
+          @event_queue = Queue.new
+          @stopped     = false
+          @timers      = {}
+
+          @subscriptions = Hash.new { |h,k| h[k] = [] }
+          @subscriptions[:__add_timer] << Proc.new { |t| set_timer(t) }
+          @subscriptions[:__add_event] << Proc.new { |e, blk| @subscriptions[e] << blk }
+        end
+
+        def set_timer(timer)
+          existing_timer = @timers[timer.event]
+
+          if existing_timer
+            elapsed_interval = Time.now - existing_timer.last_interval_start
+            timer.advance(elapsed_interval)
+          end
+
+          @timers[timer.event] = timer
+
+          fire_timer(timer)
+        end
+
+        def next_timeout
+          return nil if @timers.empty?
+          timeout = @timers.values.map(&:next_fire_time).min - Time.now
+          timeout < 0 ? 0 : timeout
+        end
+
+        def stopped?
+          @stopped
+        end
+
+        def stop
+          @stopped = true
+          wakeup
+        end
+
+        def run
+          ::TingYun::Agent.logger.debug "Running event loop"
+          while !stopped?
+            run_once
+          end
+        end
+
+        def run_once(nonblock=false)
+          wait_to_run(nonblock)
+
+          prune_timers
+          fire_timers
+
+          until @event_queue.empty?
+            evt, args = @event_queue.pop
+            dispatch_event(evt, args)
+            reschedule_timer_for_event(evt)
+          end
+        end
+
+        def wait_to_run(nonblock)
+          timeout = nonblock ? 0 : next_timeout
+          ready = IO.select([@self_pipe_rd], nil, nil, timeout)
+
+          if ready && ready[0] && ready[0][0] && ready[0][0] == @self_pipe_rd
+            @self_pipe_rd.read(1)
+          end
+        end
+
+        def fire_timers
+          @timers.each do |event, timer|
+            fire_timer(timer)
+          end
+        end
+
+        def fire_timer(timer)
+          if timer.due?
+            @event_queue << [timer.event]
+            timer.set_fired_time
+          end
+        end
+
+        def prune_timers
+          @timers.delete_if { |e, t| t.finished? }
+        end
+
+        def dispatch_event(event, args)
+          TingYun::Agent.logger.debug("EventLoop: Dispatching event '#{event}' with #{@subscriptions[event].size} callback(s).")
+
+          errors = []
+          @subscriptions[event].each do |s|
+            begin
+              s.call(*args)
+            rescue TingYun::Support::Exception::ExpiredConfigurationException, TingYun::Support::Exception::InvalidDataTokenException, TingYun::Support::Exception::InvalidDataException
+              raise
+            rescue => e
+              errors << e
+            end
+          end
+
+          if !errors.empty?
+            ::TingYun::Agent.logger.error "#{errors.size} error(s) running task for event '#{event}' in Agent Event Loop:", *errors
+          end
+        end
+
+        def reschedule_timer_for_event(e)
+          @timers[e].reschedule if @timers[e]
+        end
+
+        def on(event, &blk)
+          fire(:__add_event, event, blk)
+        end
+
+        def fire(event, *args)
+          @event_queue << [event, args]
+          wakeup
+        end
+
+        def fire_every(interval, event)
+          ::TingYun::Agent.logger.debug "Firing event #{event} every #{interval} seconds."
+          fire(:__add_timer, Timer.new(interval, event, true))
+        end
+
+        def fire_after(interval, event)
+          ::TingYun::Agent.logger.debug "Firing event #{event} after #{interval} seconds."
+          fire(:__add_timer, Timer.new(interval, event, false))
+        end
+
+        def wakeup
+          begin
+            @self_pipe_wr.write_nonblock '.'
+          rescue Errno::EAGAIN
+            ::TingYun::Agent.logger.debug "Failed to wakeup event loop"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/instance_methods/connect.rb

@@ -0,0 +1,164 @@
+# encoding: utf-8
+# This file is distributed under Ting Yun's license terms.
+require 'ting_yun/support/exception'
+require 'ting_yun/support/hostname'
+require 'ting_yun/configuration/server_source'
+require 'ting_yun/agent/instance_methods/handle_errors'
+require 'ting_yun/environment_report'
+
+
+module TingYun
+  module Agent
+    module InstanceMethods
+      module Connect
+
+        include HandleErrors
+
+        # number of attempts we've made to contact the server
+        attr_accessor :connect_attempts
+
+        # Disconnect just sets connected to false, which prevents
+        # the agent from trying to connect again
+        def disconnect
+          @connect_state = :disconnected
+          true
+        end
+
+        def connected?
+          @connect_state == :connected
+        end
+
+        def disconnected?
+          @connect_state == :disconnected
+        end
+
+        # Don't connect if we're already connected, or if we tried to connect
+        # and were rejected with prejudice because of a license issue, unless
+        # we're forced to by force_reconnect.
+        def should_connect?(force=false)
+          force || (!connected? && !disconnected?)
+        end
+
+        # Retry period is a minute for each failed attempt that
+        # we've made. This should probably do some sort of sane TCP
+        # backoff to prevent hammering the server, but a minute for
+        # each attempt seems to work reasonably well.
+        def connect_retry_period
+          [600, connect_attempts * 60].min
+        end
+
+        def note_connect_failure
+          self.connect_attempts += 1
+        end
+
+        def generate_environment_report
+          @environment_report = environment_for_connect
+        end
+
+        # We've seen objects in the environment report (Rails.env in
+        # particular) that can't seralize to JSON. Cope with that here and
+        # clear out so downstream code doesn't have to check again.
+        def sanitize_environment_report
+          if !@service.valid_to_marshal?(@environment_report)
+            @environment_report = {}
+          end
+        end
+
+
+        # Checks whether we should send environment info, and if so,
+        # returns the snapshot from the local environment.
+        # Generating the EnvironmentReport has the potential to trigger
+        # require calls in Rails environments, so this method should only
+        # be called synchronously from on the main thread.
+        def environment_for_connect
+          ::TingYun::Agent.config[:send_environment_info] ? TingYun::EnvironmentReport.new.data : {}
+        end
+
+
+        # Initializes the hash of settings that we send to the
+        # server. Returns a literal hash containing the options
+        def connect_settings
+          sanitize_environment_report
+          settings = {
+              :pid => $$,
+              :port => ::TingYun::Agent.config[:port],
+              :host => local_host,
+              :appName => ::TingYun::Agent.config.app_names,
+              :language => 'Ruby',
+              :agentVersion => ::TingYun::VERSION::STRING,
+              :env => @environment_report,
+              :config => ::TingYun::Agent.config.to_collector_hash
+          }
+          settings
+        end
+
+        def local_host
+          TingYun::Support::Hostname.get
+        end
+
+        # Returns connect data passed back from the server
+        def connect_to_server
+          @service.connect(connect_settings)
+        end
+
+        #merge server config
+        def query_server_for_configuration
+          finish_setup(connect_to_server)
+        end
+
+        # * <tt>:keep_retrying => false</tt> to only try to connect once, and
+        #   return with the connection set to nil.  This ensures we may try again
+        #   later (default true).
+        # * <tt>force_reconnect => true</tt> if you want to establish a new connection
+        #   to the server before running the worker loop.  This means you get a separate
+        #   agent run and Ting Yun sees it as a separate instance (default is false).
+        def catch_errors
+          yield
+
+        rescue TingYun::Support::Exception::ExpiredConfigurationException => e
+          handle_force_restart(e)
+          retry
+        rescue TingYun::Support::Exception::InvalidDataTokenException => e
+          handle_force_restart(e)
+          retry
+        rescue TingYun::Support::Exception::InvalidDataException => e
+          handle_server_error(e)
+        rescue => e
+          handle_other_error(e)
+        end
+
+        # Takes a hash of configuration data returned from the
+        # server and uses it to set local variables and to
+        # initialize various parts of the agent that are configured
+        # separately.
+        #
+        def finish_setup(config_data)
+          return if config_data == nil
+
+          if config_data['config']
+            ::TingYun::Agent.logger.debug "Using config from server"
+          end
+          ::TingYun::Agent.logger.debug "Server provided config: #{config_data.inspect}"
+          server_config = TingYun::Configuration::ServerSource.new(config_data)
+          ::TingYun::Agent.config.replace_or_add_config(server_config)
+          #log_connection!(config_data)
+        end
+
+        def log_connection!(config_data)
+          ::TingYun::Agent.logger.debug "Connected to TingYun Service at #{@service.collector.name}"
+          ::TingYun::Agent.logger.debug "Application Run       = #{@service.applicationId}."
+          ::TingYun::Agent.logger.debug "Connection data = #{config_data.inspect}"
+          if config_data['messages'] && config_data['messages'].any?
+            log_collector_messages(config_data['messages'])
+          end
+        end
+
+        def log_collector_messages(messages)
+          messages.each do |message|
+            ::TingYun::Agent.logger.send(message['level'].downcase, message['message'])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/instance_methods/container_data_manager.rb

@@ -0,0 +1,137 @@
+# encoding: utf-8
+require 'ting_yun/support/exception'
+require 'ting_yun/agent/collector/stats_engine'
+require 'ting_yun/agent/collector/error_collector'
+
+require 'ting_yun/agent/collector/transaction_sampler'
+
+require 'ting_yun/agent/collector/sql_sampler'
+
+
+module TingYun
+  module Agent
+    module InstanceMethods
+      module ContainerDataManager
+
+
+        attr_reader :stats_engine, :error_collector, :transaction_sampler, :sql_sampler, :middleware
+
+
+        def drop_buffered_data
+          @stats_engine.reset!
+          @transaction_sampler.reset!
+          @sql_sampler.reset!
+          @error_collector.reset!
+        end
+
+        def reset_objects_with_locks
+          init_containers
+        end
+
+
+        def init_containers
+          @stats_engine = TingYun::Agent::Collector::StatsEngine.new
+          @error_collector = TingYun::Agent::Collector::ErrorCollector.new
+          @transaction_sampler = TingYun::Agent::Collector::TransactionSampler.new
+          @sql_sampler = TingYun::Agent::Collector::SqlSampler.new
+          @middleware = TingYun::Agent::Collector::MiddleWareCollector.new(@events)
+        end
+
+        def container_for_endpoint(endpoint)
+          case endpoint
+            when :metric_data then
+              @stats_engine
+            # type code here
+          end
+        end
+
+        def transmit_data
+          ::TingYun::Agent.logger.debug('Sending data to Ting Yun Service')
+
+          @events.notify(:middleware_harvest)
+          @service.session do # use http keep-alive
+            harvest_and_send_errors
+            harvest_and_send_external_errors
+            harvest_and_send_timeslice_data
+            harvest_and_send_transaction_traces
+            harvest_and_send_slowest_sql
+          end
+        end
+
+        def harvest_and_send_timeslice_data
+          harvest_and_send_from_container(@stats_engine, :metric_data)
+        end
+
+        def harvest_and_send_errors
+          harvest_and_send_from_container(@error_collector.error_trace_array, :error_data)
+        end
+
+        def harvest_and_send_external_errors
+          harvest_and_send_from_container(@error_collector.external_error_array, :external_error_data)
+        end
+
+
+        def harvest_and_send_transaction_traces
+          harvest_and_send_from_container(@transaction_sampler, :action_trace_data)
+
+        end
+
+        def harvest_and_send_slowest_sql
+          harvest_and_send_from_container(@sql_sampler, :sql_trace)
+
+        end
+
+        # Harvests data from the given container, sends it to the named endpoint
+        # on the service, and automatically merges back in upon a recoverable
+        # failure.
+        #
+        # The given container should respond to:
+        #
+        #  #harvest!
+        #    returns an enumerable collection of data items to be sent to the
+        #    collector.
+        #
+        #  #reset!
+        #    drop any stored data and reset to a clean state.
+        #
+        #  #merge!(items)
+        #    merge the given items back into the internal buffer of the
+        #    container, so that they may be harvested again later.
+        #
+        def harvest_and_send_from_container(container, endpoint)
+          items = harvest_from_container(container, endpoint)
+          send_data_to_endpoint(endpoint, items, container)
+        end
+
+        def harvest_from_container(container, endpoint)
+          items =[]
+          begin
+             if TingYun::Agent.config[:'nbs.agent_enabled']
+               items = container.harvest!
+             else
+               container.reset!
+             end
+          rescue => e
+            TingYun::Agent.logger.error("Failed to harvest #{endpoint} data, resetting. Error: ", e)
+            container.reset!
+          end
+          items
+        end
+
+        def send_data_to_endpoint(endpoint, items, container)
+          TingYun::Agent.logger.info("Sending #{items.size} items to #{endpoint}")
+          begin
+            @service.send(endpoint, items)
+          rescue => e
+            TingYun::Agent.logger.info("Unable to send #{endpoint} data, will try again later. Error: ", e)
+            container.merge!(items)
+            raise
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/ting_yun/agent/instance_methods/handle_errors.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+# This file is distributed under Ting Yun's license terms.
+
+#
+
+module TingYun
+  module Agent
+    module InstanceMethods
+      module HandleErrors
+        # When the server sends us an error with the license key, we
+        # want to tell the user that something went wrong, and let
+        # them know where to go to get a valid license key
+        #
+        # After this runs, it disconnects the agent so that it will
+        # no longer try to connect to the server, saving the
+        # application and the server load
+        def handle_license_error(error)
+          TingYun::Agent.logger.error(\
+              error.message, \
+              "You need to obtain a valid license key, or to upgrade your account.")
+          disconnect
+        end
+
+        def handle_unrecoverable_agent_error(error)
+          TingYun::Agent.logger.error(error.message)
+          disconnect
+          shutdown
+        end
+
+        # When we have a problem connecting to the server, we need
+        # to tell the user what happened, since this is not an error
+        # we can handle gracefully.
+        def log_error(error)
+          TingYun::Agent.logger.error "Error establishing connection with Ting Yun Service at #{service.inspect}:", error
+        end
+
+        # Handles an unknown error in the worker thread by logging
+        # it and disconnecting the agent, since we are now in an
+        # unknown state.
+        def handle_other_error(error)
+          TingYun::Agent.logger.error "Unhandled error in worker thread, disconnecting this agent process:"
+          # These errors are fatal (that is, they will prevent the agent from
+          # reporting entirely), so we really want backtraces when they happen
+          TingYun::Agent.logger.log_exception(:error, error)
+          disconnect
+        end
+
+        # Handles the case where the server tells us to restart -
+        # this clears the data, clears connection attempts, and
+        # waits a while to reconnect.
+        def handle_force_restart(error)
+          TingYun::Agent.logger.debug error.message
+          drop_buffered_data
+          @service.force_restart if @service
+          @connect_state = :pending
+          sleep 30
+        end
+        def handle_force_disconnect(error)
+          TingYun::Agent.logger.warn "Ting Yun forced this agent to disconnect (#{error.message})"
+          disconnect
+        end
+
+        def handle_server_error(error)
+          TingYun::Agent.logger.error(error.message)
+          drop_buffered_data
+        end
+
+      end
+    end
+  end
+end