mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/event_buffer.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+# EventBuffer is intended to be an abstract base class. It should not be
+# instantiated directly. Subclasses should define an `append_event` method
+# looking something like this:
+#
+# def append_event(x)
+#   <attempt to append>
+#   if append_success?
+#     return x
+#   else
+#     return nil
+#   end
+# end
+
+module Mmtrix
+  module Agent
+    class EventBuffer
+      attr_reader :capacity
+
+      def initialize(capacity)
+        @capacity = capacity
+        @items    = []
+        @seen     = 0
+      end
+
+      def reset!
+        @items = []
+        @seen  = 0
+      end
+
+      def capacity=(new_capacity)
+        @capacity = new_capacity
+        old_items = @items
+        @items    = []
+        old_seen  = @seen
+        old_items.each { |i| append(i) }
+        @seen     = old_seen
+      end
+
+      def append(x)
+        @seen += 1
+        append_event(x)
+      end
+
+      def <<(x)
+        append(x)
+        self # return self for method chaining
+      end
+
+      def full?
+        @items.size >= @capacity
+      end
+
+      def size
+        @items.size
+      end
+
+      def note_dropped
+        @seen += 1
+      end
+
+      def num_seen
+        @seen
+      end
+
+      def num_dropped
+        @seen - @items.size
+      end
+
+      def sample_rate
+        @seen > 0 ? (size.to_f / @seen) : 0.0
+      end
+
+      def to_a
+        @items.dup
+      end
+
+    end
+  end
+end
+

data/lib/mmtrix/agent/event_listener.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix::Agent
+  # Basic mechanism for the agent instance to provide agent-wide eventing.
+  # It is intended to keep different pieces of the app decoupled from each other.
+  #
+  # While an EventListener could be used elsewhere, it's strongly expected
+  # your eventing needs should be met by the agent's instance.
+  class EventListener
+
+    attr_accessor :runaway_threshold
+
+    def initialize
+      @events = {}
+      @runaway_threshold = 100
+    end
+
+    def subscribe(event, &handler)
+      @events[event] ||= []
+      @events[event] << handler
+      check_for_runaway_subscriptions(event)
+    end
+
+    def check_for_runaway_subscriptions(event)
+      count = @events[event].size
+      Mmtrix::Agent.logger.debug("Run-away event subscription on #{event}? Subscribed #{count}") if count > @runaway_threshold
+    end
+
+    def clear
+      @events.clear
+    end
+
+    def notify(event, *args)
+      return unless @events.has_key?(event)
+
+      @events[event].each do |handler|
+        begin
+          handler.call(*args)
+        rescue => err
+          Mmtrix::Agent.logger.debug("Failure during notify for #{event}", err)
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/event_loop.rb

@@ -0,0 +1,193 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'thread'
+
+module Mmtrix
+  module Agent
+    class EventLoop
+      class Timer
+        attr_reader :next_fire_time, :event, :interval, :last_fired_at
+
+        def initialize(interval, event, repeat=false)
+          @interval      = interval
+          @event         = event
+          @repeat        = repeat
+          @started_at    = Time.now
+          @last_fired_at = nil
+          reschedule
+        end
+
+        def reschedule
+          @next_fire_time = calculate_next_fire_time
+        end
+
+        def advance(amount)
+          @next_fire_time -= amount
+        end
+
+        def last_interval_start
+          @last_fired_at || @started_at
+        end
+
+        def calculate_next_fire_time
+          now = Time.now
+          return now if @interval == 0
+          fire_time = @last_fired_at || now
+          while fire_time <= now
+            fire_time += @interval
+          end
+          fire_time
+        end
+
+        def set_fired_time
+          @last_fired_at = Time.now
+        end
+
+        def due?(now=Time.now)
+          now >= @next_fire_time
+        end
+
+        def finished?
+          !@repeat && @last_fired_at
+        end
+      end
+
+      def initialize
+        @self_pipe_rd, @self_pipe_wr = IO.pipe
+        @event_queue = Queue.new
+        @stopped     = false
+        @timers      = {}
+
+        @subscriptions = Hash.new { |h,k| h[k] = [] }
+        @subscriptions[:__add_timer] << Proc.new { |t| set_timer(t) }
+        @subscriptions[:__add_event] << Proc.new { |e, blk| @subscriptions[e] << blk }
+      end
+
+      def set_timer(timer)
+        existing_timer = @timers[timer.event]
+
+        if existing_timer
+          elapsed_interval = Time.now - existing_timer.last_interval_start
+          timer.advance(elapsed_interval)
+        end
+
+        @timers[timer.event] = timer
+
+        fire_timer(timer)
+      end
+
+      def next_timeout
+        return nil if @timers.empty?
+        timeout = @timers.values.map(&:next_fire_time).min - Time.now
+        timeout < 0 ? 0 : timeout
+      end
+
+      def stopped?
+        @stopped
+      end
+
+      def stop
+        @stopped = true
+        wakeup
+      end
+
+      def run
+        ::Mmtrix::Agent.logger.debug "Running event loop"
+        while !stopped?
+          run_once
+        end
+      end
+
+      def run_once(nonblock=false)
+        wait_to_run(nonblock)
+
+        prune_timers
+        fire_timers
+
+        until @event_queue.empty?
+          evt, args = @event_queue.pop
+          dispatch_event(evt, args)
+          reschedule_timer_for_event(evt)
+        end
+      end
+
+      def wait_to_run(nonblock)
+        timeout = nonblock ? 0 : next_timeout
+        ready = IO.select([@self_pipe_rd], nil, nil, timeout)
+
+        if ready && ready[0] && ready[0][0] && ready[0][0] == @self_pipe_rd
+          @self_pipe_rd.read(1)
+        end
+      end
+
+      def fire_timers
+        @timers.each do |event, timer|
+          fire_timer(timer)
+        end
+      end
+
+      def fire_timer(timer)
+        if timer.due?
+          @event_queue << [timer.event]
+          timer.set_fired_time
+        end
+      end
+
+      def prune_timers
+        @timers.delete_if { |e, t| t.finished? }
+      end
+
+      def dispatch_event(event, args)
+        Mmtrix::Agent.logger.debug("EventLoop: Dispatching event '#{event}' with #{@subscriptions[event].size} callback(s).")
+
+        errors = []
+        @subscriptions[event].each do |s|
+          begin
+            s.call(*args)
+          rescue Mmtrix::Agent::ForceRestartException, Mmtrix::Agent::ForceDisconnectException
+            raise
+          rescue => e
+            errors << e
+          end
+        end
+
+        if !errors.empty?
+          ::Mmtrix::Agent.logger.error "#{errors.size} error(s) running task for event '#{event}' in Agent Event Loop:", *errors
+        end
+      end
+
+      def reschedule_timer_for_event(e)
+        @timers[e].reschedule if @timers[e]
+      end
+
+      def on(event, &blk)
+        fire(:__add_event, event, blk)
+      end
+
+      def fire(event, *args)
+        @event_queue << [event, args]
+        wakeup
+      end
+
+      def fire_every(interval, event)
+        ::Mmtrix::Agent.logger.debug "Firing event #{event} every #{interval} seconds."
+        fire(:__add_timer, Timer.new(interval, event, true))
+      end
+
+      def fire_after(interval, event)
+        ::Mmtrix::Agent.logger.debug "Firing event #{event} after #{interval} seconds."
+        fire(:__add_timer, Timer.new(interval, event, false))
+      end
+
+      def wakeup
+        begin
+          @self_pipe_wr.write_nonblock '.'
+        rescue Errno::EAGAIN
+          ::Mmtrix::Agent.logger.debug "Failed to wakeup event loop"
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/harvester.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    class Harvester
+
+      attr_accessor :starting_pid
+
+      # Inject target for after_fork call to avoid spawning thread in tests
+      def initialize(events, after_forker=Mmtrix::Agent)
+        # We intentionally don't set our pid as started at this point.
+        # Startup routines must call mark_started when they consider us set!
+        @starting_pid = nil
+        @after_forker = after_forker
+
+        if events
+          events.subscribe(:start_transaction, &method(:on_transaction))
+        end
+      end
+
+      def on_transaction(*_)
+        return unless restart_in_children_enabled? &&
+                        needs_restart? &&
+                        harvest_thread_enabled?
+
+        restart_harvest_thread
+      end
+
+      def mark_started(pid = Process.pid)
+        @starting_pid = pid
+      end
+
+      def needs_restart?(pid = Process.pid)
+        @starting_pid != pid
+      end
+
+      def restart_in_children_enabled?
+        Mmtrix::Agent.config[:restart_thread_in_children]
+      end
+
+      def harvest_thread_enabled?
+        !Mmtrix::Agent.config[:disable_harvest_thread]
+      end
+
+      def restart_harvest_thread
+        @after_forker.after_fork(:force_reconnect => true)
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/hash_extensions.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module HashExtensions
+      module_function
+
+      # recurses through hashes and arrays and stringifies keys
+      def stringify_keys_in_object(object)
+        case object
+        when Hash
+          object.inject({}) do |memo, (k, v)|
+            memo[k.to_s] = stringify_keys_in_object(v)
+            memo
+          end
+        when Array
+          object.map {|o| stringify_keys_in_object(o)}
+        else
+          object
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/hostname.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Hostname
+      def self.get
+        dyno_name = ENV['DYNO']
+        if dyno_name && ::Mmtrix::Agent.config[:'heroku.use_dyno_names']
+          matching_prefix = heroku_dyno_name_prefix(dyno_name)
+          dyno_name = "#{matching_prefix}.*" if matching_prefix
+          dyno_name
+        else
+          Socket.gethostname
+        end
+      end
+
+      def self.heroku_dyno_name_prefix(dyno_name)
+        get_dyno_prefixes.find do |dyno_prefix|
+          dyno_name.start_with?(dyno_prefix + ".")
+        end
+      end
+
+      def self.get_dyno_prefixes
+        ::Mmtrix::Agent.config[:'heroku.dyno_name_prefixes_to_shorten']
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/http_clients/curb_wrappers.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module HTTPClients
+
+      class CurbRequest
+        def initialize( curlobj )
+          @curlobj = curlobj
+        end
+
+        def type
+          'Curb'
+        end
+
+        def host
+          self["host"] || self["Host"] || self.uri.host
+        end
+
+        def method
+          @curlobj._nr_http_verb
+        end
+
+        def []( key )
+          @curlobj.headers[ key ]
+        end
+
+        def []=( key, value )
+          @curlobj.headers[ key ] = value
+        end
+
+        def uri
+          @uri ||= Mmtrix::Agent::HTTPClients::URIUtil.parse_url(@curlobj.url)
+        end
+      end
+
+
+      class CurbResponse
+
+        def initialize(curlobj)
+          @headers = {}
+          @curlobj = curlobj
+        end
+
+        def [](key)
+          @headers[ key.downcase ]
+        end
+
+        def to_hash
+          @headers.dup
+        end
+
+        def append_header_data( data )
+          key, value = data.split( /:\s*/, 2 )
+          @headers[ key.downcase ] = value
+          @curlobj._nr_header_str ||= ''
+          @curlobj._nr_header_str << data
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/mmtrix/agent/http_clients/excon_wrappers.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module HTTPClients
+      class ExconHTTPResponse
+        def initialize(response)
+          @response = response
+          # Since HTTP headers are case-insensitive, we normalize all of them to
+          # upper case here, and then also in our [](key) implementation.
+          @normalized_headers = {}
+          headers = response.respond_to?(:headers) ? response.headers : response[:headers]
+          (headers || {}).each do |key, val|
+            @normalized_headers[key.upcase] = val
+          end
+        end
+
+        def [](key)
+          @normalized_headers[key.upcase]
+        end
+
+        def to_hash
+          @normalized_headers.dup
+        end
+      end
+
+      class ExconHTTPRequest
+        def initialize(datum)
+          @datum = datum
+
+          @method = @datum[:method].to_s.upcase
+          @scheme = @datum[:scheme]
+          @port   = @datum[:port]
+          @path   = @datum[:path]
+
+          headers = @datum[:headers]
+          if hostname = (headers['host'] || headers['Host'])
+            @host = hostname.split(':').first
+          else
+            @host = @datum[:host]
+          end
+        end
+
+        attr_reader :host, :method
+
+        def type
+          "Excon"
+        end
+
+        def [](key)
+          @datum[:headers][key]
+        end
+
+        def []=(key, value)
+          @datum[:headers] ||= {}
+          @datum[:headers][key] = value
+        end
+
+        def uri
+          URI.parse("#{@scheme}://#{@host}:#{@port}#{@path}")
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/http_clients/httpclient_wrappers.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module HTTPClients
+      class HTTPClientResponse
+        attr_reader :response
+
+        def initialize(response)
+          @response = response
+        end
+
+        def [](key)
+          response.headers.each do |k,v|
+            if key.downcase == k.downcase
+              return v
+            end
+          end
+          nil
+        end
+
+        def to_hash
+          response.headers
+        end
+      end
+
+      class HTTPClientRequest
+        attr_reader :request, :uri
+
+        def initialize(request)
+          @request = request
+          @uri = request.header.request_uri
+        end
+
+        def type
+          "HTTPClient"
+        end
+
+        def method
+          request.header.request_method
+        end
+
+        def host
+          if hostname = (self['host'] || self['Host'])
+            hostname.split(':').first
+          else
+            uri.host.to_s
+          end
+        end
+
+        def [](key)
+          request.headers[key]
+        end
+
+        def []=(key, value)
+          request.http_header[key] = value
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/http_clients/net_http_wrappers.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module HTTPClients
+      class NetHTTPRequest
+        def initialize(connection, request)
+          @connection = connection
+          @request = request
+        end
+
+        def type
+          'Net::HTTP'
+        end
+
+        def host
+          if hostname = self['host']
+            hostname.split(':').first
+          else
+            @connection.address
+          end
+        end
+
+        def method
+          @request.method
+        end
+
+        def [](key)
+          @request[key]
+        end
+
+        def []=(key, value)
+          @request[key] = value
+        end
+
+        def uri
+          case @request.path
+          when /^https?:\/\//
+            URI(@request.path)
+          else
+            scheme = @connection.use_ssl? ? 'https' : 'http'
+            URI("#{scheme}://#{@connection.address}:#{@connection.port}#{@request.path}")
+          end
+        end
+      end
+    end
+  end
+end