scout_apm 2.2.0.pre3 → 2.3.0.pre

data/lib/scout_apm/remote/recorder.rb

@@ -0,0 +1,57 @@
+module ScoutApm
+  module Remote
+    class Recorder
+      attr_reader :logger
+      attr_reader :remote_agent_host
+      attr_reader :remote_agent_port
+
+      def initialize(remote_agent_host, remote_agent_port, logger)
+        @remote_agent_host = remote_agent_host
+        @remote_agent_port = remote_agent_port
+        @logger = logger
+      end
+
+      def start
+        # nothing to do
+        self
+      end
+
+      def stop
+        # nothing to do
+      end
+
+      def record!(request)
+        begin
+          t1 = Time.now
+          # Mark this request as recorded, so the next lookup on this thread, it
+          # can be recreated
+          request.recorded!
+
+          # Only send requests that we actually want. Incidental http &
+          # background thread stuff can just be dropped
+          unless request.job? || request.web?
+            return
+          end
+
+          request.prepare_to_dump!
+          message = ScoutApm::Remote::Message.new('record', 'record!', request)
+          encoded = message.encode
+          logger.debug "Remote Agent: Posting a message of length: #{encoded.length}"
+          post(encoded)
+          t2 = Time.now
+
+          logger.debug("Remote Recording took: #{t2.to_f - t1.to_f} seconds")
+        rescue => e
+          logger.debug "Remote: Error while sending to collector: #{e.inspect}, #{e.backtrace.join("\n")}"
+        end
+      end
+
+      def post(encoded_message)
+        http = Net::HTTP.new(remote_agent_host, remote_agent_port)
+        request = Net::HTTP::Post.new("/users")
+        request.body = encoded_message
+        response = http.request(request)
+      end
+    end
+  end
+end

data/lib/scout_apm/remote/router.rb

@@ -0,0 +1,49 @@
+module ScoutApm
+  module Remote
+    class Router
+      attr_reader :logger
+      attr_reader :routes
+
+      # If/When we add different types, this signature should change to a hash
+      # of {type => Object}, rather than building it in the initializer here.
+      #
+      # Keys of routes should be strings
+      def initialize(recorder, logger)
+        @routes = {
+          'record' => recorder
+        }
+
+        @logger = logger
+      end
+
+      # A message is a 2 element array [:type, :command, [args]].
+      # For this first creation, this should be ['record', 'record', [TrackedRequest]] (the args arg should always be an array, even w/ only 1 item)
+      #
+      # Where
+      #   type: ['recorder']
+      #   command: any function supported on that type of object
+      #   args: any array of arguments
+      #
+      # Raises on unknown message
+      #
+      # Returns whatever the recipient object returns
+      def handle(msg)
+        message = Remote::Message.decode(msg)
+        assert_type(message)
+        call_route(message)
+      end
+
+      private
+
+      def assert_type(message)
+        if ! routes.keys.include?(message.type.to_s)
+          raise "Unknown type: #{message.type.to_s}"
+        end
+      end
+
+      def call_route(message)
+        routes[message.type].send(message.command, *message.args)
+      end
+    end
+  end
+end

data/lib/scout_apm/remote/server.rb

@@ -0,0 +1,58 @@
+# Web Server bound to localhost that listens for remote agent reports. Forwards
+# onto the router
+module ScoutApm
+  module Remote
+    class Server
+      attr_reader :router
+      attr_reader :bind
+      attr_reader :port
+      attr_reader :logger
+
+      def initialize(bind, port, router, logger)
+        @router = router
+        @logger = logger
+        @bind = bind
+        @port = port
+        @server = nil
+      end
+
+      def start
+        @server = WEBrick::HTTPServer.new(
+          :BindAddress => bind,
+          :Port => port,
+          :AccessLog => [],
+          :Logger => @logger
+        )
+
+        @server.mount_proc '/' do |request, response|
+          router.handle(request.body)
+
+          # arbitrary response, client doesn't expect anything in particular
+          response.body = 'Ok'
+        end
+
+        @thread = Thread.new do
+          begin
+            logger.debug("Remote: Starting Server on #{bind}:#{port}")
+
+            @server.start
+
+            logger.debug("Remote: Server returned after #start call, thread exiting")
+          rescue => e
+            logger.debug("Remote: Server Exception, #{e}")
+          end
+        end
+      end
+
+      def running?
+        @thread.alive?
+        @server && @server.status == :Running
+      end
+
+      def stop
+        @server.stop
+        @thread.kill
+      end
+    end
+  end
+end

data/lib/scout_apm/reporter.rb

@@ -17,31 +17,36 @@ module ScoutApm
       @instant_key = instant_key
     end
 
-    # TODO: Parse & return a real response object, not the HTTP Response object
     def report(payload, headers = {})
-      # Some posts (typically ones under development) bypass the ingestion pipeline and go directly to the webserver. They use direct_host instead of host
-      hosts = [:deploy_hook, :instant_trace].include?(type) ? config.value('direct_host') : config.value('host')
+      hosts = determine_hosts
 
-      Array(hosts).each do |host|
-        full_uri = uri(host)
-        response = post(full_uri, payload, headers)
-        unless response && response.is_a?(Net::HTTPSuccess)
-          logger.warn "Error on checkin to #{full_uri.to_s}: #{response.inspect}"
-        end
+      if config.value('compress_payload')
+        original_payload_size = payload.length
+
+        payload, compression_headers = compress_payload(payload)
+        headers.merge!(compression_headers)
+
+        compress_payload_size = payload.length
+        ScoutApm::Agent.instance.logger.debug("Original Size: #{original_payload_size} Compressed Size: #{compress_payload_size}")
       end
+
+      post_payload(hosts, payload, headers)
     end
 
     def uri(host)
+      encoded_app_name = CGI.escape(Environment.instance.application_name)
+      key = config.value('key')
+
       case type
       when :checkin
-        URI.parse("#{host}/apps/checkin.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/checkin.scout?key=#{key}&name=#{encoded_app_name}")
       when :app_server_load
-        URI.parse("#{host}/apps/app_server_load.scout?key=#{config.value('key')}&name=#{CGI.escape(Environment.instance.application_name)}")
+        URI.parse("#{host}/apps/app_server_load.scout?key=#{key}&name=#{encoded_app_name}")
       when :deploy_hook
-        URI.parse("#{host}/apps/deploy.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}")
+        URI.parse("#{host}/apps/deploy.scout?key=#{key}&name=#{encoded_app_name}")
       when :instant_trace
-        URI.parse("#{host}/apps/instant_trace.scout?key=#{config.value('key')}&name=#{CGI.escape(config.value('name'))}&instant_key=#{instant_key}")
-      end.tap{|u| logger.debug("Posting to #{u.to_s}")}
+        URI.parse("#{host}/apps/instant_trace.scout?key=#{key}&name=#{encoded_app_name}&instant_key=#{instant_key}")
+      end.tap { |u| logger.debug("Posting to #{u}") }
     end
 
     def can_report?
@@ -106,7 +111,10 @@ module ScoutApm
     # Net::HTTP::Proxy returns a regular Net::HTTP class if the first argument (host) is nil.
     def http(url)
       proxy_uri = URI.parse(config.value('proxy').to_s)
-      http = Net::HTTP::Proxy(proxy_uri.host,proxy_uri.port,proxy_uri.user,proxy_uri.password).new(url.host, url.port)
+      http = Net::HTTP::Proxy(proxy_uri.host,
+                              proxy_uri.port,
+                              proxy_uri.user,
+                              proxy_uri.password).new(url.host, url.port)
       if url.is_a?(URI::HTTPS)
         http.use_ssl = true
         http.ca_file = CA_FILE
@@ -114,5 +122,33 @@ module ScoutApm
       end
       http
     end
+
+    def compress_payload(payload)
+      [
+        ScoutApm::Utils::GzipHelper.new.deflate(payload),
+        { 'Content-Encoding' => 'gzip' }
+      ]
+    end
+
+    # Some posts (typically ones under development) bypass the ingestion
+    # pipeline and go directly to the webserver. They use direct_host instead
+    # of host
+    def determine_hosts
+      if [:deploy_hook, :instant_trace].include?(type)
+        config.value('direct_host')
+      else
+        config.value('host')
+      end
+    end
+
+    def post_payload(hosts, payload, headers)
+      Array(hosts).each do |host|
+        full_uri = uri(host)
+        response = post(full_uri, payload, headers)
+        unless response && response.is_a?(Net::HTTPSuccess)
+          logger.warn "Error on checkin to #{full_uri}: #{response.inspect}"
+        end
+      end
+    end
   end
 end

data/lib/scout_apm/request_histograms.rb

@@ -39,6 +39,10 @@ module ScoutApm
       initialize_histograms_hash
     end
 
+    def raw(item)
+      @histograms[item]
+    end
+
     def initialize_histograms_hash
       @histograms = Hash.new { |h, k| h[k] = NumericHistogram.new(histogram_size) }
     end

data/lib/scout_apm/request_manager.rb

@@ -11,7 +11,7 @@ module ScoutApm
     def self.find
       req = Thread.current[:scout_request]
 
-      if req && req.recorded?
+      if req && (req.stopping? || req.recorded?)
         nil
       else
         req
@@ -25,6 +25,7 @@ module ScoutApm
               else
                 ScoutApm::FakeStore.new
               end
+
       Thread.current[:scout_request] = TrackedRequest.new(store)
     end
   end

data/lib/scout_apm/scored_item_set.rb

@@ -63,6 +63,12 @@ module ScoutApm
       end
     end
 
+    # Equal to another set only if exactly the same set of items is inside
+    def eql?(other)
+      items == other.items
+    end
+
+    alias :== :eql?
 
     private
 
@@ -75,5 +81,6 @@ module ScoutApm
         items[new_item.name] = [new_item.score, new_item.call]
       end
     end
+
   end
 end

data/lib/scout_apm/serializers/db_query_serializer_to_json.rb

@@ -0,0 +1,15 @@
+module ScoutApm
+  module Serializers
+    class DbQuerySerializerToJson
+      attr_reader :db_query_metrics
+
+      def initialize(db_query_metrics)
+        @db_query_metrics = db_query_metrics
+      end
+
+      def as_json
+        db_query_metrics.map{|metric| metric.as_json }
+      end
+    end
+  end
+end

data/lib/scout_apm/serializers/histograms_serializer_to_json.rb

@@ -0,0 +1,21 @@
+
+module ScoutApm
+  module Serializers
+    class HistogramsSerializerToJson
+      attr_reader :histograms
+
+      def initialize(histograms)
+        @histograms = histograms
+      end
+
+      def as_json
+        histograms.map do |histo|
+          {
+            "name" => histo.name,
+            "histogram" => histo.histogram.as_json,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -2,9 +2,9 @@
 module ScoutApm
   module Serializers
     class PayloadSerializer
-      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+      def self.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
         if ScoutApm::Agent.instance.config.value("report_format") == 'json'
-          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+          ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
         else
           metadata = metadata.dup
           metadata.default = nil
@@ -15,7 +15,14 @@ module ScoutApm
                        :metrics           => metrics,
                        :slow_transactions => slow_transactions,
                        :jobs              => jobs,
-                       :slow_jobs         => slow_jobs)
+                       :slow_jobs         => slow_jobs,
+
+                       # as_json returns a ruby object. Since it's not a simple
+                       # array, use this to maintain compatibility with json
+                       # payloads. At this point, the marshal code branch is
+                       # very rarely used anyway.
+                       :histograms        => HistogramsSerializerToJson.new(histograms).as_json,
+                       :db_query_metrics  => db_query_metrics)
         end
       end
 

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -2,7 +2,7 @@ module ScoutApm
   module Serializers
     module PayloadSerializerToJson
       class << self
-        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
+        def serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms, db_query_metrics)
           metadata.merge!({:payload_version => 2})
 
           jsonify_hash({:metadata => metadata,
@@ -10,7 +10,11 @@ module ScoutApm
                         :slow_transactions => rearrange_the_slow_transactions(slow_transactions),
                         :jobs => JobsSerializerToJson.new(jobs).as_json,
                         :slow_jobs => SlowJobsSerializerToJson.new(slow_jobs).as_json,
-                      })
+                        :histograms => HistogramsSerializerToJson.new(histograms).as_json,
+                        :db_metrics => {
+                          :query => DbQuerySerializerToJson.new(db_query_metrics).as_json,
+                        },
+          })
         end
 
         # For the old style of metric serializing.
@@ -67,10 +71,6 @@ module ScoutApm
             %Q["#{formatee.iso8601}"]
           when nil
             "null"
-          when TrueClass
-            "true"
-          when FalseClass
-            "false"
           else # strings and everything
             %Q["#{escape(formatee)}"]
           end

data/lib/scout_apm/serializers/slow_jobs_serializer_to_json.rb

@@ -21,9 +21,11 @@ module ScoutApm
             "allocations" => job.allocations,
             "seconds_since_startup" => job.seconds_since_startup,
             "hostname" => job.hostname,
+            "git_sha" => job.git_sha,
             "metrics" => MetricsToJsonSerializer.new(job.metrics).as_json, # New style of metrics
             "allocation_metrics" => MetricsToJsonSerializer.new(job.allocation_metrics).as_json, # New style of metrics
             "context" => job.context.to_hash,
+            "truncated_metrics" => job.truncated_metrics,
 
             "score" => job.score,
           }
@@ -32,4 +34,3 @@ module ScoutApm
     end
   end
 end
-

data/lib/scout_apm/server_integrations/puma.rb

@@ -24,10 +24,13 @@ module ScoutApm
       end
 
       def install
-        ::Puma.cli_config.options[:before_worker_boot] << Proc.new do
+        old = ::Puma.cli_config.options[:before_worker_boot] || []
+        new = Array(old) + [Proc.new do
           logger.info "Installing Puma worker loop."
           ScoutApm::Agent.instance.start_background_worker
-        end
+        end]
+
+        ::Puma.cli_config.options[:before_worker_boot] = new
       rescue
         logger.warn "Unable to install Puma worker loop: #{$!.message}"
       end