spartan_apm 0.0.0.rc1

data/lib/spartan_apm/instrumentation.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  # Code for instrumenting other classes with capture blocks to capture how long
+  # calling specified methods take.
+  module Instrumentation
+    class << self
+      # Apply all the bundled instrumenters.
+      def auto_instrument!
+        ActiveRecord.new.tap { |instance| instance.instrument! if instance.valid? }
+        Bunny.new.tap { |instance| instance.instrument! if instance.valid? }
+        Cassandra.new.tap { |instance| instance.instrument! if instance.valid? }
+        Curb.new.tap { |instance| instance.instrument! if instance.valid? }
+        Dalli.new.tap { |instance| instance.instrument! if instance.valid? }
+        Elasticsearch.new.tap { |instance| instance.instrument! if instance.valid? }
+        Excon.new.tap { |instance| instance.instrument! if instance.valid? }
+        HTTPClient.new.tap { |instance| instance.instrument! if instance.valid? }
+        HTTP.new.tap { |instance| instance.instrument! if instance.valid? }
+        NetHTTP.new.tap { |instance| instance.instrument! if instance.valid? }
+        Redis.new.tap { |instance| instance.instrument! if instance.valid? }
+        Typhoeus.new.tap { |instance| instance.instrument! if instance.valid? }
+      end
+
+      # Instrument a class by surrounding specified instance methods with capture blocks.
+      def instrument!(klass, name, methods, exclusive: false, module_name: nil)
+        # Create a module that will be prepended to the specified class.
+        unless module_name
+          camelized_name = name.to_s.gsub(/[^a-z0-9]+([a-z0-9])/i) { |m| m[m.length - 1, m.length].upcase }
+          camelized_name = "#{camelized_name[0].upcase}#{camelized_name[1, camelized_name.length]}"
+          module_name = "#{klass.name.split("::").join}#{camelized_name}Instrumentation"
+        end
+        if const_defined?(module_name)
+          raise ArgumentError.new("#{name} has alrady been instrumented in #{klass.name}")
+        end
+
+        # The method of overriding kwargs changed in ruby 2.7
+        ruby_major, ruby_minor, _ = RUBY_VERSION.split(".").collect(&:to_i)
+        ruby_3_args = (ruby_major >= 3 || (ruby_major == 2 && ruby_minor >= 7))
+        splat_args = (ruby_3_args ? "..." : "*args, &block")
+
+        # Dark arts & witchery to dynamically generate the module methods.
+        instrumentation_module = const_set(module_name, Module.new)
+        Array(methods).each do |method_name|
+          instrumentation_module.class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            def #{method_name}(#{splat_args})
+              SpartanAPM.capture(#{name.to_sym.inspect}, exclusive: #{exclusive.inspect}) do
+                super(#{splat_args})
+              end
+            end
+          RUBY
+        end
+
+        klass.prepend(instrumentation_module)
+      end
+    end
+  end
+end
+
+require_relative "instrumentation/base"
+require_relative "instrumentation/active_record"
+require_relative "instrumentation/bunny"
+require_relative "instrumentation/cassandra"
+require_relative "instrumentation/curb"
+require_relative "instrumentation/dalli"
+require_relative "instrumentation/elasticsearch"
+require_relative "instrumentation/excon"
+require_relative "instrumentation/httpclient"
+require_relative "instrumentation/http"
+require_relative "instrumentation/net_http"
+require_relative "instrumentation/redis"
+require_relative "instrumentation/typhoeus"

data/lib/spartan_apm/measure.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  # This class holds the metrics captured during a request. Metrics are captured
+  # in one minute increments and then written to Redis.
+  class Measure
+    attr_reader :app, :action, :timers, :counts, :error, :error_message, :error_backtrace
+
+    @mutex = Mutex.new
+    @current_measures = nil
+    @last_bucket = nil
+    @string_cache = StringCache.new
+    @error_cache = Concurrent::Map.new
+
+    class << self
+      # Get the list of Measures stored for the current minute time bucket. Every minute
+      # this list is started anew. This method will also automatically kick off the process
+      # to persist Measures from the previous time bucket to Redis if necessary.
+      # @return [Concurrent::Array<Measure>]
+      def current_measures
+        bucket = SpartanAPM.bucket(Time.now)
+        last_bucket = @last_bucket
+        if bucket != last_bucket
+          persist_measures = nil
+          @mutex.synchronize do
+            # This check is made again within the mutex block so that we don't have
+            # to lock the mutex every time we make the check if the bucket has changed.
+            if bucket != @last_bucket
+              persist_measures = @current_measures
+              @last_bucket = bucket
+              @current_measures = Concurrent::Array.new
+              @string_cache = StringCache.new
+              @error_cache = {}
+            end
+          end
+          if persist_measures && !persist_measures.empty?
+            if SpartanAPM.persist_asynchronously?
+              Thread.new { Persistence.store!(last_bucket, persist_measures) }
+            else
+              Persistence.store!(last_bucket, persist_measures)
+            end
+          end
+        end
+
+        @current_measures
+      end
+
+      # @api private
+      # Used for consistency in test cases.
+      def clear_current_measures!
+        @mutex.synchronize do
+          @current_measures = nil
+          @last_bucket = nil
+        end
+      end
+
+      # Flush all currently enqueued Measures to Redis.
+      def flush
+        return if @last_bucket.nil?
+        bucket = nil
+        measures = nil
+        @mutex.synchronize do
+          bucket = @last_bucket
+          measures = @current_measures
+          @current_measures = Concurrent::Array.new
+          @string_cache = StringCache.new
+          @error_cache = {}
+        end
+        unless measures.empty?
+          Persistence.store!(bucket, measures)
+        end
+      end
+
+      # @api private
+      #
+      # Fetch error information from a cache. The cache is used since backtraces
+      # can be quite long and, if the application got into a bad state, a lot of
+      # errors could be generated in a short time. The cache is here to prevent
+      # memory bloat if that happens by only storing one copy of each error trace.
+      #
+      # There is also a hard limit of 1000 distinct errors at a time. This is a
+      # protection in case errors end up with dynamically generated traces so that
+      # don't use up all the memory. If your application gets over 1000 distinct
+      # errors in a minute, seeing a truncated list of them is the least of your
+      # worries.
+      def error_cache_fetch(error)
+        return nil unless error
+        backtrace = SpartanAPM.clean_backtrace(error.backtrace)
+        error_key = Digest::MD5.hexdigest("#{error.class.name} #{backtrace&.join}")
+        cached_error = @error_cache[error_key]
+        unless cached_error
+          return nil if @error_cache.size > 1000
+          cached_error = [error.class.name, error.message, backtrace]
+          @error_cache[error_key] = cached_error
+        end
+        cached_error
+      end
+
+      attr_reader :string_cache
+    end
+
+    def initialize(app, action = nil)
+      @app = self.class.string_cache.fetch(app)
+      @action = self.class.string_cache.fetch(action) if action
+      @timers = Hash.new(0.0)
+      @counts = Hash.new(0)
+      @current_name = nil
+      @current_start_time = nil
+      @current_exclusive = false
+    end
+
+    def action=(value)
+      @action = self.class.string_cache.fetch(value)
+    end
+
+    def app=(value)
+      @app = self.class.string_cache.fetch(value)
+    end
+
+    # Capture the timing for a component. See SpartanAPM#capture for more info.
+    def capture(name, exclusive: false)
+      name = name.to_sym
+      if @current_exclusive
+        # Already capturing from within an exclusive block, so don't interrupt that capture.
+        yield
+      else
+        start_time = Time.now
+        restore_name = @current_name
+        if restore_name
+          @timers[restore_name] += start_time - @current_start_time
+        end
+        @current_name = name
+        @current_start_time = start_time
+        @current_exclusive = exclusive
+        begin
+          yield
+        ensure
+          end_time = Time.now
+          @timers[name] += end_time - @current_start_time
+          @counts[name] += 1
+          if restore_name
+            @current_name = restore_name
+            @current_start_time = end_time
+          else
+            @current_name = nil
+            @current_start_time = nil
+          end
+        end
+      end
+    end
+
+    # Capture the timing for a component. See SpartanAPM#capture_time for more info.
+    def capture_time(name, elapsed_time)
+      name = name.to_sym
+      @timers[name] += elapsed_time.to_f
+      @counts[name] += 1
+    end
+
+    # Capture an error. See SpartanAPM#capture_error for more info.
+    def capture_error(error)
+      @error, @error_message, @error_backtrace = self.class.error_cache_fetch(error)
+    end
+
+    # This method must be called to add the Measure to the measures for
+    # the current bucket.
+    def record!
+      if (action && !@timers.empty?) || error
+        self.class.current_measures << self
+      end
+    end
+  end
+end

data/lib/spartan_apm/metric.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  # Data structure for information about the request metrics for a particular time.
+  class Metric
+    attr_reader :time
+    attr_accessor :count, :avg, :p50, :p90, :p99, :error_count, :components
+
+    def initialize(time)
+      @time = time
+      @components = {}
+    end
+
+    def component_names
+      @components.keys.collect { |n| n.to_s.freeze }
+    end
+
+    def component_request_time(name)
+      Array(@components[name.to_s])[0]
+    end
+
+    def component_request_count(name)
+      Array(@components[name.to_s])[1]
+    end
+  end
+end

data/lib/spartan_apm/middleware/rack/end_middleware.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  module Middleware
+    module Rack
+      # Middleware that should be added of the end of the middleware chain.
+      class EndMiddleware
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          start_time = Time.now
+          SpartanAPM.capture(:app) do
+            begin
+              @app.call(env)
+            ensure
+              # Capture how much time was spent in middleware.
+              middleware_start_time = env["spartan_apm.middleware_start_time"]
+              if middleware_start_time
+                SpartanAPM.capture_time(:middleware, start_time.to_f - middleware_start_time)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spartan_apm/middleware/rack/start_middleware.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  module Middleware
+    module Rack
+      # Middleware that should be added to the start of the start of the middleware chain.
+      class StartMiddleware
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          if SpartanAPM.ignore_request?("web", env["PATH_INFO"])
+            @app.call(env)
+          else
+            start_time = Time.now.to_f
+
+            # This value is used in EndMiddleware to capture how long all the middleware
+            # between the two middlewares took to execute.
+            env["spartan_apm.middleware_start_time"] = start_time
+
+            SpartanAPM.measure("web") do
+              begin
+                @app.call(env)
+              ensure
+                # Record how long the web request was enqueued before the Rack server
+                # got the request if that information is available.
+                enqueued_at_time = request_queue_start_time(env)
+                if enqueued_at_time && start_time > enqueued_at_time
+                  SpartanAPM.capture_time(:queue, start_time - enqueued_at_time)
+                end
+              end
+            end
+          end
+        end
+
+        private
+
+        def request_queue_start_time(env)
+          # There's a few of differing conventions on where web servers record the
+          # start time on a request in the proxied HTTP headers.
+          header = (env["HTTP_X_REQUEST_START"] || env["HTTP_X_QUEUE_START"])
+          start_time = nil
+          if header
+            header = header[2, header.size] if header.start_with?("t=")
+            t = header.to_f
+            # Header could be in seconds, milliseconds, or microseconds
+            t /= 1000.0 if t > 5_000_000_000
+            t /= 1000.0 if t > 5_000_000_000
+            start_time = t if t > 0
+          end
+          start_time
+        end
+      end
+    end
+  end
+end

data/lib/spartan_apm/middleware/sidekiq/end_middleware.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  module Middleware
+    module Sidekiq
+      # Middleware that should be added of the end of the middleware chain.
+      class EndMiddleware
+        def call(worker, msg, queue, &block)
+          start_time = Time.now.to_f
+          SpartanAPM.capture(:app) do
+            begin
+              yield
+            ensure
+              # Capture how much time was spent in middleware.
+              middleware_start_time = msg["spartan_apm.middleware_start_time"]
+              if middleware_start_time
+                SpartanAPM.capture_time(:middleware, start_time - middleware_start_time)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spartan_apm/middleware/sidekiq/start_middleware.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  module Middleware
+    module Sidekiq
+      # Middleware that should be added to the start of the start of the middleware chain.
+      class StartMiddleware
+        def call(worker, msg, queue, &block)
+          if SpartanAPM.ignore_request?("sidekiq", worker.class.name)
+            yield
+          else
+            start_time = Time.now.to_f
+
+            # This value is used in EndMiddleware to capture how long all the middleware
+            # between the two middlewares took to execute.
+            msg["spartan_apm.middleware_start_time"] = start_time
+
+            SpartanAPM.measure("sidekiq", worker.class.name) do
+              begin
+                yield
+              ensure
+                # Capture how long the message was enqueued in Redis before a worker got the job.
+                enqueued_time = msg["enqueued_at"].to_f if msg.is_a?(Hash)
+                if enqueued_time && enqueued_time > 0 && start_time > enqueued_time
+                  SpartanAPM.capture_time(:queue, start_time - enqueued_time)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spartan_apm/middleware.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SpartanAPM
+  module Middleware
+    module Rack
+    end
+
+    module Sidekiq
+    end
+  end
+end
+
+require_relative "middleware/rack/end_middleware"
+require_relative "middleware/rack/start_middleware"
+require_relative "middleware/sidekiq/end_middleware"
+require_relative "middleware/sidekiq/start_middleware"