newrelic_rpm 3.5.6.55 → 3.5.7.57.beta

data/CHANGELOG

@@ -1,6 +1,15 @@
 
 # New Relic Ruby Agent Release Notes #
 
+## v3.5.7 ##
+
+  * Resolved some issues with tracking of frontend queue time, particularly
+    when the agent is running on an app hosted on Heroku.  The agent will now
+    more reliably parse the headers described in
+    https://newrelic.com/docs/features/tracking-front-end-time and will
+    automatically detect whether the times provided are in seconds,
+    milliseconds or microseconds.
+
 ## v3.5.6 ##
 
   * Use HTTPS by default

data/lib/new_relic/agent/instrumentation/controller_instrumentation.rb

@@ -438,22 +438,20 @@ module NewRelic
           end
         end
 
-        include NewRelic::Agent::Instrumentation::QueueTime
-
         # Return a Time instance representing the upstream start time.
         # now is a Time instance to fall back on if no other candidate
         # for the start time is found.
         def _detect_upstream_wait(now)
-          queue_start = nil
           if newrelic_request_headers
-            queue_start = parse_frontend_headers(newrelic_request_headers)
+            queue_start = QueueTime.parse_frontend_timestamp(newrelic_request_headers)
+            QueueTime.record_frontend_metrics(queue_start, now) if queue_start
           end
           queue_start || now
         rescue => e
           ::NewRelic::Agent.logger.error("Error detecting upstream wait time:", e)
           now
         end
-        
+
         # returns the NewRelic::MethodTraceStats object associated
         # with the dispatcher time measurement
         def _dispatch_stat

data/lib/new_relic/agent/instrumentation/queue_time.rb

@@ -1,210 +1,54 @@
 module NewRelic
   module Agent
     module Instrumentation
+      # https://newrelic.com/docs/features/tracking-front-end-time
+      # Record queue time metrics based on any of three headers
+      # which can be set on the request.
       module QueueTime
-        unless defined?(MAIN_HEADER)
-          MAIN_HEADER = 'HTTP_X_REQUEST_START'
-          MIDDLEWARE_HEADER = 'HTTP_X_MIDDLEWARE_START'
-          QUEUE_HEADER = 'HTTP_X_QUEUE_START'
-          ALT_QUEUE_HEADER = 'HTTP_X_QUEUE_TIME'
-          HEROKU_QUEUE_HEADER = 'HTTP_X_HEROKU_QUEUE_WAIT_TIME'
-          APP_HEADER = 'HTTP_X_APPLICATION_START'
-
-          HEADER_REGEX = /([^\s\/,(t=)]+)? ?t=([0-9\.]+)/
-          SERVER_METRIC = 'WebFrontend/WebServer/'
-          MIDDLEWARE_METRIC = 'Middleware/'
-          # no individual queue metric - more than one queue?!
-          ALL_SERVER_METRIC = 'WebFrontend/WebServer/all'
-          ALL_MIDDLEWARE_METRIC = 'Middleware/all'
-          ALL_QUEUE_METRIC = 'WebFrontend/QueueTime'
-        end
-
-        def parse_frontend_headers(headers)
-          # these methods add internal state, so we dup so other parts
-          # of the app don't have to worry about it.
-          # May have performance implications with very large env hashes
-          env = headers.dup
-          add_end_time_header(Time.now, env)
-          middleware_start = parse_middleware_time_from(env)
-          queue_start = parse_queue_time_from(env)
-          server_start = parse_server_time_from(env)
-          # returned for the controller instrumentation
-          [middleware_start, queue_start, server_start].min
-        end
-
-        private
-
-        # main method to extract server time info from env hash,
-        # records individual server metrics and one roll-up for all servers
-        def parse_server_time_from(env)
-          end_time = parse_end_time(env)
-          matches = get_matches_from_header(MAIN_HEADER, env)
-
-          record_individual_server_stats(end_time, matches)
-          record_rollup_server_stat(end_time, matches)
-        end
-
-        def parse_middleware_time_from(env)
-          end_time = parse_end_time(env)
-          matches = get_matches_from_header(MIDDLEWARE_HEADER, env)
-
-          record_individual_middleware_stats(end_time, matches)
-          oldest_time = record_rollup_middleware_stat(end_time, matches)
-          # notice this bit: we reset the end time to the earliest
-          # middleware tag so that other frontend metrics don't
-          # include this time.
-          add_end_time_header(oldest_time, env)
-          oldest_time
-        end
-
-        def parse_queue_time_from(env)
-          oldest_time = nil
-          end_time = parse_end_time(env)
-          alternate_length = check_for_alternate_queue_length(env)
-          if alternate_length
-            # skip all that fancy-dan stuff
-            NewRelic::Agent.get_stats(ALL_QUEUE_METRIC).trace_call(alternate_length)
-            oldest_time = (end_time - alternate_length) # should be a time
-          else
-            matches = get_matches_from_header(QUEUE_HEADER, env)
-            oldest_time = record_rollup_queue_stat(end_time, matches)
+        unless defined?(REQUEST_START_HEADER)
+          REQUEST_START_HEADER    = 'HTTP_X_REQUEST_START'
+          QUEUE_START_HEADER      = 'HTTP_X_QUEUE_START'
+          QUEUE_DURATION_HEADER   = 'HTTP_X_QUEUE_TIME'
+          MIDDLEWARE_START_HEADER = 'HTTP_X_MIDDLEWARE_START'
+          ALL_QUEUE_METRIC        = 'WebFrontend/QueueTime'
+          # any timestamps before this are thrown out and the parser
+          # will try again with a larger unit (2000/1/1 UTC)
+          EARLIEST_ACCEPTABLE_TIMESTAMP = 946684800
+        end
+
+        module_function
+
+        def parse_frontend_timestamp(headers)
+          candidate_headers = [ REQUEST_START_HEADER, QUEUE_START_HEADER,
+                                MIDDLEWARE_START_HEADER ]
+          candidate_headers.map do |header|
+            if headers[header]
+              parse_timestamp(timestamp_string_from_header_value(headers[header]))
+            end
+          end.compact.min
+        end
+
+        def record_frontend_metrics(start_time, now=Time.now)
+          NewRelic::Agent.instance.stats_engine.get_stats(ALL_QUEUE_METRIC) \
+            .record_data_point((now - start_time).to_f)
+        end
+
+        def timestamp_string_from_header_value(value)
+          case value
+          when /^\s*([\d+\.]+)\s*$/ then $1
+          # following regexp intentionally unanchored to handle
+          # (ie ignore) leading server names
+          when /t=([\d+\.]+)/       then $1
           end
-          # notice this bit: we reset the end time to the earliest
-          # queue tag or the start time minus the queue time so that
-          # other frontend metrics don't include this time.
-          add_end_time_header(oldest_time, env)
-          oldest_time
-        end
-
-        def check_for_alternate_queue_length(env)
-          heroku_length = check_for_heroku_queue_length(env)
-          return heroku_length if heroku_length
-          header = env[ALT_QUEUE_HEADER]
-          return nil unless header
-          (header.gsub('t=', '').to_i / 1_000_000.0)
-        end
-
-        def check_for_heroku_queue_length(env)
-          header = env[HEROKU_QUEUE_HEADER]
-          return nil unless header
-          (header.gsub(/[^0-9]/, '').to_i / 1_000.0)
-        end
-
-        def get_matches_from_header(header, env)
-          return [] if env.nil?
-          get_matches(env[header]).map do |name, time|
-            convert_to_name_time_pair(name, time.sub('.', ''))
-          end
-        end
-
-        def get_matches(string)
-          string.to_s.scan(HEADER_REGEX)
-        end
-
-        def convert_to_name_time_pair(name, time)
-          [name, convert_from_microseconds(time.to_i)]
-        end
-
-        def record_individual_stat_of_type(type, end_time, matches)
-          matches = matches.sort_by {|name, time| time }
-          matches.reverse!
-          matches.inject(end_time) {|end_time, pair|
-            name, time = pair
-            self.send(type, name, time, end_time) if name
-            time
-          }
-        end
-
-        # goes through the list of servers and records each one in
-        # reverse order, subtracting the time for each successive
-        # server from the earlier ones in the list.
-        # an example because it's complicated:
-        # start data:
-        # [['a', Time.at(1000)], ['b', Time.at(1001)]], start time: Time.at(1002)
-        # initial run: Time.at(1002), ['b', Time.at(1001)]
-        # next: Time.at(1001), ['a', Time.at(1000)]
-        # see tests for more
-        def record_individual_server_stats(end_time, matches) # (Time, [[String, Time]]) -> nil
-          record_individual_stat_of_type(:record_server_time_for, end_time, matches)
         end
 
-        def record_individual_middleware_stats(end_time, matches)
-          record_individual_stat_of_type(:record_middleware_time_for, end_time, matches)
-        end
-
-        # records the total time for all servers in a rollup metric
-        def record_rollup_server_stat(end_time, matches) # (Time, [String, Time]) -> nil
-          record_rollup_stat_of_type(ALL_SERVER_METRIC, end_time, matches)
-        end
-
-        def record_rollup_middleware_stat(end_time, matches)
-          record_rollup_stat_of_type(ALL_MIDDLEWARE_METRIC, end_time, matches)
-        end
-
-        def record_rollup_queue_stat(end_time, matches)
-          record_rollup_stat_of_type(ALL_QUEUE_METRIC, end_time, matches)
-        end
-
-        def record_rollup_stat_of_type(metric, end_time, matches)
-          oldest_time = find_oldest_time(matches) || end_time
-          record_time_stat(metric, oldest_time, end_time)
-          oldest_time
-        end
-
-        # searches for the first server to touch a request
-        def find_oldest_time(matches) # [[String, Time]] -> Time
-          matches.map do |name, time|
-            time
-          end.min
-        end
-
-        # basically just assembles the metric name
-        def record_server_time_for(name, start_time, end_time) # (Maybe String, Time, Time) -> nil
-          record_time_stat(SERVER_METRIC + name, start_time, end_time) if name
-        end
-
-        def record_middleware_time_for(name, start_time, end_time)
-          record_time_stat(MIDDLEWARE_METRIC + name, start_time, end_time)
-        end
-
-        # Checks that the time is not negative, and does the actual
-        # data recording
-        def record_time_stat(name, start_time, end_time) # (String, Time, Time) -> nil
-          total_time = end_time - start_time
-          if total_time < 0
-            raise "should not provide an end time less than start time: #{end_time.strftime('%s.%N')} is less than #{start_time.strftime('%s.%N')}. total time is #{total_time}."
-          else
-            NewRelic::Agent.get_stats(name).trace_call(total_time)
-          end
-        end
-
-        def add_end_time_header(end_time, env) # (Time, Env) -> nil
-          return unless end_time
-          env[APP_HEADER] = "t=#{convert_to_microseconds(end_time)}"
-        end
-
-        def parse_end_time(env)
-          header = env[APP_HEADER]
-          return Time.now unless header
-          convert_from_microseconds(header.gsub('t=', '').to_i)
-        end
-
-        # convert a time to the value provided by the header, for convenience
-        def convert_to_microseconds(time) # Time -> Int
-          raise TypeError.new('Cannot convert a non-time into microseconds') unless time.is_a?(Time) || time.is_a?(Numeric)
-          return time if time.is_a?(Numeric)
-          (time.to_f * 1_000_000).to_i
-        end
-
-        # convert a time from the header value (time in microseconds)
-        # into a ruby time object
-        def convert_from_microseconds(int) # Int -> Time
-          raise TypeError.new('Cannot convert a non-number into a time') unless int.is_a?(Time) || int.is_a?(Numeric)
-          return int if int.is_a?(Time)
-          Time.at((int / 1_000_000.0))
+        def parse_timestamp(string)
+          cut_off = Time.at(EARLIEST_ACCEPTABLE_TIMESTAMP)
+          [1_000_000, 1_000, 1].map do |divisor|
+            Time.at(string.to_f / divisor)
+          end.find {|candidate| candidate > cut_off }
         end
       end
     end
   end
 end
-

data/lib/new_relic/version.rb

@@ -8,7 +8,7 @@ module NewRelic
 
     MAJOR = 3
     MINOR = 5
-    TINY  = 6
+    TINY  = 7
 
     begin
       require File.join(File.dirname(__FILE__), 'build')

data/test/multiverse/suites/rails/queue_time_test.rb

@@ -43,8 +43,8 @@ class QueueTimeTest < ActionDispatch::IntegrationTest
     assert extract_queue_time_from_response > 0, "Queue time was missing or zero"
   end
 
-  def get_queued(header="HTTP_X_QUEUE_START")
-    get('/queue/queued', nil, \
+  def get_queued(header="HTTP_X_REQUEST_START")
+    get('/queue/queued', nil,
         header => "t=#{(Time.now.to_i * 1_000_000) - 1_000}")
   end
 

data/test/multiverse/suites/sinatra/sinatra_test.rb

@@ -57,12 +57,8 @@ class SinatraTest < Test::Unit::TestCase
   end
 
   def test_queue_time_headers_are_passed_to_agent
-    get '/user/login', {}, {"X-Request-Start" => 't=1234567890'}
-    metric_names = ::NewRelic::Agent.agent.stats_engine.stats_hash.keys.map{|k| k.name}
-    assert metric_names.include?("Middleware/all")
-    assert metric_names.include?("WebFrontend/QueueTime")
-    assert metric_names.include?("WebFrontend/WebServer/all")
-    assert ::NewRelic::Agent.agent.stats_engine.get_stats("WebFrontend/WebServer/all")
+    get '/user/login', {}, { 'HTTP_X_REQUEST_START' => 't=1360973845' }
+    assert ::NewRelic::Agent.agent.stats_engine.lookup_stats('WebFrontend/QueueTime')
   end
 
   def test_shown_errors_get_caught

data/test/new_relic/agent/agent_test_controller_test.rb

@@ -84,7 +84,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
 
   def test_new_queue_integration
     # make this test deterministic
-    Time.stubs(:now => Time.at(2))
+    Time.stubs(:now => Time.at(1360973845))
 
     NewRelic::Agent::AgentTestController.clear_headers
     engine.clear_stats
@@ -95,35 +95,9 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
     check_metric_time('WebFrontend/QueueTime', 1, 0.1)
   end
 
-
-  def test_new_middleware_integration
-    # make this test deterministic
-    Time.stubs(:now => Time.at(2))
-
-    engine.clear_stats
-    start = ((Time.now - 1).to_f * 1_000_000).to_i
-    NewRelic::Agent::AgentTestController.set_some_headers 'HTTP_X_MIDDLEWARE_START'=> "t=#{start}"
-    get :index
-
-    check_metric_time('Middleware/all', 1, 0.1)
-  end
-
-  def test_new_server_time_integration
-    # make this test deterministic
-    Time.stubs(:now => Time.at(2))
-
-    NewRelic::Agent::AgentTestController.clear_headers
-    engine.clear_stats
-    start = ((Time.now - 1).to_f * 1_000_000).to_i
-    NewRelic::Agent::AgentTestController.set_some_headers 'HTTP_X_REQUEST_START'=> "t=#{start}"
-    get :index
-
-    check_metric_time('WebFrontend/WebServer/all', 1, 0.1)
-  end
-
   def test_new_frontend_work_integration
     # make this test deterministic
-    Time.stubs(:now => Time.at(10))
+    Time.stubs(:now => Time.at(1360973845))
 
     engine.clear_stats
     times = [Time.now - 3, Time.now - 2, Time.now - 1]
@@ -132,10 +106,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
                                                             'HTTP_X_REQUEST_START'=> "t=#{times[0]}", 'HTTP_X_QUEUE_START' => "t=#{times[1]}", 'HTTP_X_MIDDLEWARE_START' => "t=#{times[2]}"})
     get :index
 
-
-    check_metric_time('WebFrontend/WebServer/all', 1, 0.1)
-    check_metric_time('Middleware/all', 1, 0.1)
-    check_metric_time('WebFrontend/QueueTime', 1, 0.1)
+    check_metric_time('WebFrontend/QueueTime', 3, 0.1)
   end
 
   def test_render_inline
@@ -160,10 +131,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
                 'Apdex/new_relic/agent/agent_test/action_with_error',
                 'HttpDispatcher',
                 'Controller/new_relic/agent/agent_test/action_with_error',
-                'Errors/all',
-                'Middleware/all',
-                'WebFrontend/WebServer/all',
-                'WebFrontend/QueueTime']
+                'Errors/all']
 
     compare_metrics metrics, engine.metrics.reject{|m| m.index('Response')==0 || m.index('CPU')==0}
     assert_equal 1, engine.get_stats_no_scope("Controller/new_relic/agent/agent_test/action_with_error").call_count
@@ -184,10 +152,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
                 'Apdex/new_relic/agent/agent_test/action_with_error',
                 'HttpDispatcher',
                 'Controller/new_relic/agent/agent_test/action_with_error',
-                'Errors/all',
-                'WebFrontend/QueueTime',
-                'Middleware/all',
-                'WebFrontend/WebServer/all']
+                'Errors/all']
 
     compare_metrics metrics, engine.metrics.reject{|m| m.index('Response')==0 || m.index('CPU')==0}
     assert_equal 1, engine.get_stats_no_scope("Controller/new_relic/agent/agent_test/action_with_error").call_count
@@ -208,10 +173,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
                 'Apdex/new_relic/agent/agent_test/action_with_before_filter_error',
                 'HttpDispatcher',
                 'Controller/new_relic/agent/agent_test/action_with_before_filter_error',
-                'Errors/all',
-                'WebFrontend/QueueTime',
-                'Middleware/all',
-                'WebFrontend/WebServer/all']
+                'Errors/all']
 
     compare_metrics metrics, engine.metrics.reject{|m| m.index('Response')==0 || m.index('CPU')==0 || m.index('GC')==0}
     assert_equal 1, engine.get_stats_no_scope("Controller/new_relic/agent/agent_test/action_with_before_filter_error").call_count
@@ -344,7 +306,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
 
   def test_queue_headers_apache
     # make this test deterministic
-    Time.stubs(:now => Time.at(10))
+    Time.stubs(:now => Time.at(1360973845))
 
     NewRelic::Agent::AgentTestController.clear_headers
     engine.clear_stats
@@ -363,7 +325,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
   end
   def test_queue_headers_heroku
     # make this test deterministic
-    Time.stubs(:now => Time.at(10))
+    Time.stubs(:now => Time.at(1360973845))
 
     engine.clear_stats
     NewRelic::Agent::AgentTestController.clear_headers
@@ -383,7 +345,7 @@ class NewRelic::Agent::AgentTestControllerTest < ActionController::TestCase
 
   def test_queue_headers_heroku_queue_length
     # make this test deterministic
-    Time.stubs(:now => Time.at(10))
+    Time.stubs(:now => Time.at(1360973845))
 
     engine.clear_stats
     NewRelic::Agent::AgentTestController.clear_headers

data/test/new_relic/agent/instrumentation/controller_instrumentation_test.rb

@@ -5,30 +5,36 @@ class NewRelic::Agent::Instrumentation::ControllerInstrumentationTest < Test::Un
     include NewRelic::Agent::Instrumentation::ControllerInstrumentation
   end
 
+  def setup
+    @object = TestObject.new
+  end
+
   def test_detect_upstream_wait_basic
     start_time = Time.now
     object = TestObject.new
     # should return the start time above by default
-    object.expects(:newrelic_request_headers).returns({:request => 'headers'}).twice
-    object.expects(:parse_frontend_headers).with({:request => 'headers'}).returns(start_time)
-    assert_equal(start_time, object.send(:_detect_upstream_wait, start_time))
+    @object.expects(:newrelic_request_headers).returns({:request => 'headers'}).twice
+    NewRelic::Agent::Instrumentation::QueueTime.expects(:parse_frontend_timestamp) \
+      .with({:request => 'headers'}).returns(start_time)
+    assert_equal(start_time, @object.send(:_detect_upstream_wait, start_time))
   end
 
   def test_detect_upstream_wait_with_upstream
     start_time = Time.now
     runs_at = start_time + 1
-    object = TestObject.new
-    object.expects(:newrelic_request_headers).returns(true).twice
-    object.expects(:parse_frontend_headers).returns(start_time)
-    assert_equal(start_time, object.send(:_detect_upstream_wait, runs_at))
+    @object = TestObject.new
+    @object.expects(:newrelic_request_headers).returns(true).twice
+    NewRelic::Agent::Instrumentation::QueueTime.expects(:parse_frontend_timestamp).returns(start_time)
+    assert_equal(start_time, @object.send(:_detect_upstream_wait, runs_at))
   end
 
   def test_detect_upstream_wait_swallows_errors
     start_time = Time.now
     object = TestObject.new
     # should return the start time above when an error is raised
-    object.expects(:newrelic_request_headers).returns({:request => 'headers'}).twice
-    object.expects(:parse_frontend_headers).with({:request => 'headers'}).raises("an error")
-    assert_equal(start_time, object.send(:_detect_upstream_wait, start_time))
+    @object.expects(:newrelic_request_headers).returns({:request => 'headers'}).twice
+    NewRelic::Agent::Instrumentation::QueueTime.expects(:parse_frontend_timestamp) \
+      .with({:request => 'headers'}).raises("an error")
+    assert_equal(start_time, @object.send(:_detect_upstream_wait, start_time))
   end
 end

data/test/new_relic/agent/instrumentation/queue_time_test.rb

@@ -1,396 +1,90 @@
 require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'test_helper'))
 class NewRelic::Agent::Instrumentation::QueueTimeTest < Test::Unit::TestCase
-  require 'new_relic/agent/instrumentation/queue_time'
-  include NewRelic::Agent::Instrumentation::QueueTime
+  include NewRelic::Agent::Instrumentation
 
-  def setup
-    NewRelic::Agent.instance.stats_engine.clear_stats
+  def test_parse_frontend_timestamp_given_queue_start_header
+    header = { 'HTTP_X_QUEUE_START' => format_header_time(Time.now - 60) }
+    assert_in_delta(seconds_ago(60), QueueTime.parse_frontend_timestamp(header), 0.1)
   end
 
-  def create_test_start_time(env)
-    env[APP_HEADER] = "t=#{convert_to_microseconds(Time.at(1002))}"
+  def test_parse_frontend_timestamp_given_request_start_header
+    header = { 'HTTP_X_REQUEST_START' => format_header_time(Time.now - 60) }
+    assert_in_delta(seconds_ago(60), QueueTime.parse_frontend_timestamp(header), 0.1)
   end
 
-  def test_parse_frontend_headers
-    middleware_start = Time.at(1002)
-    queue_start = Time.at(1001)
-    server_start = Time.at(1000)
-    Time.stubs(:now).returns(Time.at(1003)) # whee!
-    self.expects(:add_end_time_header).with(Time.at(1003), {:env => 'hash'})
-    # ordering is important here, unfortunately, the mocks don't
-    # support that kind of checking.
-    self.expects(:parse_middleware_time_from).with({:env => 'hash'}).returns(middleware_start)
-    self.expects(:parse_queue_time_from).with({:env => 'hash'}).returns(queue_start)
-    self.expects(:parse_server_time_from).with({:env => 'hash'}).returns(server_start)
-    assert_equal(server_start, parse_frontend_headers({:env => 'hash'}), "should return the oldest start time")
+  def test_parse_frontend_timestamp_given_middleware_start_header
+    header = { 'HTTP_X_MIDDLEWARE_START' => format_header_time(Time.now - 60) }
+    assert_in_delta(seconds_ago(60), QueueTime.parse_frontend_timestamp(header), 0.1)
   end
 
-  def test_parse_frontend_headers_should_return_earliest_start
-    middleware_start = Time.at(1002)
-    queue_start = Time.at(1000)
-    server_start = Time.at(1001)
-    Time.stubs(:now).returns(Time.at(1003)) # whee!
-    self.expects(:add_end_time_header).with(Time.at(1003), {:env => 'hash'})
-    # ordering is important here, unfortunately, the mocks don't
-    # support that kind of checking.
-    self.expects(:parse_middleware_time_from).with({:env => 'hash'}).returns(middleware_start)
-    self.expects(:parse_queue_time_from).with({:env => 'hash'}).returns(queue_start)
-    self.expects(:parse_server_time_from).with({:env => 'hash'}).returns(server_start)
-    assert_equal(queue_start, parse_frontend_headers({:env => 'hash'}), "should return the oldest start time")
-  end
-
-  def test_all_combined_frontend_headers
-    env = {}
-    env[MAIN_HEADER] = "t=#{convert_to_microseconds(Time.at(1000))}"
-    env[QUEUE_HEADER] = "t=#{convert_to_microseconds(Time.at(1001))}"
-    env[MIDDLEWARE_HEADER] = "t=#{convert_to_microseconds(Time.at(1002))}"
-
-    env[APP_HEADER] = "t=#{convert_to_microseconds(Time.at(1003))}"
-
-    assert_calls_metrics('WebFrontend/WebServer/all', 'WebFrontend/QueueTime', 'Middleware/all') do
-      assert_equal(Time.at(1002), parse_middleware_time_from(env))
-      assert_equal(Time.at(1001), parse_queue_time_from(env))
-      assert_equal(Time.at(1000), parse_server_time_from(env))
-    end
-
-    check_metric_time('WebFrontend/WebServer/all', 1.0, 0.001)
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.001)
-    check_metric_time('Middleware/all', 1.0, 0.001)
-  end
-
-  def test_combined_middleware_and_queue
-    env = {}
-    env[QUEUE_HEADER] = "t=#{convert_to_microseconds(Time.at(1000))}"
-    env[MIDDLEWARE_HEADER] = "t=#{convert_to_microseconds(Time.at(1001))}"
-    create_test_start_time(env)
-
-    assert_calls_metrics('Middleware/all', 'WebFrontend/QueueTime') do
-      parse_middleware_time_from(env)
-      assert_equal(Time.at(1000), parse_queue_time_from(env))
-    end
-
-    check_metric_time('Middleware/all', 1.0, 0.001)
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.001)
-  end
-
-  def test_combined_queue_and_server
-    env = {}
-    env[MAIN_HEADER] = "t=#{convert_to_microseconds(Time.at(1000))}"
-    env[QUEUE_HEADER] = "t=#{convert_to_microseconds(Time.at(1001))}"
-    create_test_start_time(env)
-
-    assert_calls_metrics('WebFrontend/WebServer/all', 'WebFrontend/QueueTime') do
-      assert_equal(Time.at(1001), parse_queue_time_from(env))
-      parse_server_time_from(env)
-    end
-
-    check_metric_time('WebFrontend/WebServer/all', 1.0, 0.001)
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.001)
-  end
-
-  def test_combined_middleware_and_server
-    env = {}
-    env[MAIN_HEADER] = "t=#{convert_to_microseconds(Time.at(1000))}"
-    env[MIDDLEWARE_HEADER] = "t=#{convert_to_microseconds(Time.at(1001))}"
-    create_test_start_time(env)
-
-    assert_calls_metrics('WebFrontend/WebServer/all', 'Middleware/all') do
-      parse_middleware_time_from(env)
-      parse_server_time_from(env)
-    end
-
-    check_metric_time('WebFrontend/WebServer/all', 1.0, 0.001)
-    check_metric_time('Middleware/all', 1.0, 0.001)
-  end
-
-  # initial base case, a router and a static content server
-  def test_parse_server_time_from_initial
-    env = {}
-    create_test_start_time(env)
-    time1 = convert_to_microseconds(Time.at(1000))
-    time2 = convert_to_microseconds(Time.at(1001))
-    env['HTTP_X_REQUEST_START'] = "servera t=#{time1}, serverb t=#{time2}"
-    assert_calls_metrics('WebFrontend/WebServer/all', 'WebFrontend/WebServer/servera', 'WebFrontend/WebServer/serverb') do
-      parse_server_time_from(env)
-    end
-    check_metric_time('WebFrontend/WebServer/all', 2.0, 0.1)
-    check_metric_time('WebFrontend/WebServer/servera', 1.0, 0.1)
-    check_metric_time('WebFrontend/WebServer/serverb', 1.0, 0.1)
-  end
-
-  def test_parse_server_time_accepting_milliseconds_resolution_separator_char
-    env = {'HTTP_X_REQUEST_START' => "t=1000.000000"}
-    create_test_start_time(env)
-    env['HTTP_X_QUEUE_START'] = "t=1001.000000"
-    assert_calls_metrics('WebFrontend/WebServer/all') do
-      assert_equal(Time.at(1000), parse_server_time_from(env))
-    end
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      assert_equal(Time.at(1001), parse_queue_time_from(env))
-    end
-    check_metric_time('WebFrontend/WebServer/all', 2.0, 0.1)
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.1)
-  end
-
-  # test for backwards compatibility with old header
-  def test_parse_server_time_from_with_no_server_name
-    env = {'HTTP_X_REQUEST_START' => "t=#{convert_to_microseconds(Time.at(1001))}"}
-    create_test_start_time(env)
-    assert_calls_metrics('WebFrontend/WebServer/all') do
-      parse_server_time_from(env)
-    end
-    check_metric_time('WebFrontend/WebServer/all', 1.0, 0.1)
-  end
-
-  def test_parse_server_time_from_with_bad_header
-    env = {'HTTP_X_REQUEST_START' => 't=t=t=t='}
-    create_test_start_time(env)
-    assert_calls_metrics('WebFrontend/WebServer/all') do
-      parse_server_time_from(env)
-    end
-  end
-
-  def test_parse_server_time_from_with_no_header
-    assert_calls_metrics('WebFrontend/WebServer/all') do
-      parse_server_time_from({})
-    end
-  end
-
-  def test_parse_middleware_time
-    env = {}
-    create_test_start_time(env)
-    time1 = convert_to_microseconds(Time.at(1000))
-    time2 = convert_to_microseconds(Time.at(1001))
-
-    env['HTTP_X_MIDDLEWARE_START'] = "base t=#{time1}, second t=#{time2}"
-    assert_calls_metrics('Middleware/all', 'Middleware/base', 'Middleware/second') do
-      parse_middleware_time_from(env)
-    end
-    check_metric_time('Middleware/all', 2.0, 0.1)
-    check_metric_time('Middleware/base', 1.0, 0.1)
-    check_metric_time('Middleware/second', 1.0, 0.1)
-  end
-
-  def test_parse_queue_time
-    env = {}
-    create_test_start_time(env)
-    time1 = convert_to_microseconds(Time.at(1000))
-
-    env['HTTP_X_QUEUE_START'] = "t=#{time1}"
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      assert_equal(Time.at(1000), parse_queue_time_from(env))
-    end
+  def test_parse_frontend_timestamp_from_earliest_header
+    headers = { 'HTTP_X_REQUEST_START'    => format_header_time(Time.now - 63),
+                'HTTP_X_QUEUE_START'      => format_header_time(Time.now - 62),
+                'HTTP_X_MIDDLEWARE_START' => format_header_time(Time.now - 61)}
 
-    check_metric_time('WebFrontend/QueueTime', 2.0, 0.1)
+    assert_in_delta(seconds_ago(63), QueueTime.parse_frontend_timestamp(headers), 0.1)
   end
 
-  def test_check_for_alternate_queue_length
-    env = {}
-    create_test_start_time(env)
-    env['HTTP_X_QUEUE_TIME'] = '1000000'
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      assert_equal(Time.at(1001), parse_queue_time_from(env))
-    end
+  def test_parse_frontend_timestamp_from_earliest_header_out_of_order
+    headers = { 'HTTP_X_MIDDLEWARE_START' => format_header_time(Time.now - 63),
+                'HTTP_X_REQUEST_START'    => format_header_time(Time.now - 62),
+                'HTTP_X_QUEUE_START'      => format_header_time(Time.now - 61) }
 
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.001)
+    assert_in_delta(seconds_ago(63), QueueTime.parse_frontend_timestamp(headers), 0.1)
   end
 
-  def test_check_for_alternate_queue_length_override
-    env = {}
-    create_test_start_time(env)
-    env['HTTP_X_QUEUE_START'] = 't=1' # obviously incorrect
-    env['HTTP_X_QUEUE_TIME'] = '1000000'
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      assert_equal(Time.at(1001), parse_queue_time_from(env))
-    end
-
-    # alternate queue should override normal header
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.001)
+  def test_parse_frontend_timestamp_from_header_in_seconds
+    header = { 'HTTP_X_QUEUE_START' => "t=#{Time.now.to_f - 60}" }
+    assert_in_delta(seconds_ago(60), QueueTime.parse_frontend_timestamp(header), 0.1)
   end
 
-  def test_check_for_heroku_queue_length
-    env = {}
-    create_test_start_time(env)
-    env['HTTP_X_HEROKU_QUEUE_WAIT_TIME'] = '1000'
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      assert_equal(Time.at(1001), parse_queue_time_from(env))
-    end
-
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.001)
+  def test_parse_frontend_timestamp_from_header_in_milliseconds
+    header = { 'HTTP_X_QUEUE_START' => "t=#{(Time.now.to_f - 60) * 1_000}" }
+    assert_in_delta(seconds_ago(60), QueueTime.parse_frontend_timestamp(header), 0.1)
   end
 
-  def test_check_for_heroku_queue_length_override
-    env = {}
-    create_test_start_time(env)
-    env['HTTP_X_QUEUE_TIME'] = '10000000' # ten MEEELION useconds
-    env['HTTP_X_HEROKU_QUEUE_WAIT_TIME'] = '1000'
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      assert_equal(Time.at(1001), parse_queue_time_from(env))
-    end
-
-    # heroku queue should override alternate queue
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.001)
+  def test_parse_frontend_timestamp_from_header_with_multiple_servers
+    now = Time.now.to_f
+    header = { 'HTTP_X_QUEUE_START' => "servera t=#{now - 60}, serverb t=#{now - 30}" }
+    assert_in_delta(seconds_ago(60), QueueTime.parse_frontend_timestamp(header), 0.1)
   end
 
-  # each server should be one second, and the total would be 2 seconds
-  def test_record_individual_server_stats
-    matches = [['foo', Time.at(1000)], ['bar', Time.at(1001)]]
-    assert_calls_metrics('WebFrontend/WebServer/foo', 'WebFrontend/WebServer/bar') do
-      record_individual_server_stats(Time.at(1002), matches)
-    end
-    check_metric_time('WebFrontend/WebServer/foo', 1.0, 0.1)
-    check_metric_time('WebFrontend/WebServer/bar', 1.0, 0.1)
+  def test_parse_frontend_timestamp_from_header_missing_t_equals
+    header = { 'HTTP_X_REQUEST_START' => (Time.now.to_f - 60).to_s }
+    assert_in_delta(seconds_ago(60), QueueTime.parse_frontend_timestamp(header), 0.1)
   end
 
-  def test_record_rollup_server_stat
-    assert_calls_metrics('WebFrontend/WebServer/all') do
-      record_rollup_server_stat(Time.at(1001), [['a', Time.at(1000)]])
+  def test_recording_queue_time_metric
+    assert_metric_value_in_delta(60, 'WebFrontend/QueueTime', 0.1) do
+      QueueTime.record_frontend_metrics(Time.at(Time.now.to_f - 60))
     end
-    check_metric_time('WebFrontend/WebServer/all', 1.0, 0.1)
   end
 
-  def test_record_rollup_server_stat_no_data
-    assert_calls_metrics('WebFrontend/WebServer/all') do
-      record_rollup_server_stat(Time.at(1001), [])
-    end
-    check_metric_time('WebFrontend/WebServer/all', 0.0, 0.001)
-  end
+  def test_parsing_malformed_header
+    header = { 'HTTP_X_REQUEST_START' => 'gobledy gook' }
 
-  def test_record_rollup_middleware_stat
-    assert_calls_metrics('Middleware/all') do
-      record_rollup_middleware_stat(Time.at(1001), [['a', Time.at(1000)]])
-    end
-    check_metric_time('Middleware/all', 1.0, 0.1)
+    assert_nil QueueTime.parse_frontend_timestamp(header)
   end
 
-  def test_record_rollup_middleware_stat_no_data
-    assert_calls_metrics('Middleware/all') do
-      record_rollup_middleware_stat(Time.at(1001), [])
-    end
-    check_metric_time('Middleware/all', 0.0, 0.001)
+  def test_parse_timestamp_can_identify_unit
+    now = Time.now.to_f
+    assert_in_delta(now, QueueTime.parse_timestamp(now.to_f.to_s).to_f, 0.001)
+    assert_in_delta(now, QueueTime.parse_timestamp((now.to_f * 1_000).to_s).to_f, 0.001)
+    assert_in_delta(now, QueueTime.parse_timestamp((now.to_f * 1_000_000).to_s).to_f, 0.001)
   end
 
-  def test_record_rollup_queue_stat
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      record_rollup_queue_stat(Time.at(1001), [[nil, Time.at(1000)]])
-    end
-    check_metric_time('WebFrontend/QueueTime', 1.0, 0.1)
-  end
-
-  def test_record_rollup_queue_stat_no_data
-    assert_calls_metrics('WebFrontend/QueueTime') do
-      record_rollup_queue_stat(Time.at(1001), [])
-    end
-    check_metric_time('WebFrontend/QueueTime', 0.0, 0.001)
-  end
-
-  def test_find_oldest_time
-    test_array = [
-                   ['c', Time.at(1002)],                   
-                   ['a', Time.at(1000)],
-                   ['b', Time.at(1001)],
-                   ['d', Time.at(1000)],
-                  ]
-    assert_equal(find_oldest_time(test_array), Time.at(1000),
-                 "Should be the oldest time in the array")
+  def format_header_time(time=Time.now)
+    "t=#{(time.to_f * 1_000_000).to_i}"
   end
 
-  # trivial test but the method doesn't do much
-  def test_record_server_time_for
-    name = 'foo'
-    time = Time.at(1000)
-    start_time = Time.at(1001)
-    self.expects(:record_time_stat).with('WebFrontend/WebServer/foo', time, start_time)
-    record_server_time_for(name, time, start_time)
+  def seconds_ago(seconds)
+    Time.at(Time.now.to_f - seconds)
   end
 
-  def test_record_time_stat
-    assert_calls_metrics('WebFrontend/WebServer/foo') do
-      record_time_stat('WebFrontend/WebServer/foo', Time.at(1000), Time.at(1001))
-    end
-    check_metric_time('WebFrontend/WebServer/foo', 1.0, 0.1)
-    assert_raises(RuntimeError) do
-      record_time_stat('foo', Time.at(1001), Time.at(1000))
-    end
-  end
-
-  def test_record_time_stat_with_end_after_start
-    record_time_stat('WebFrontend/WebServer/foo', Time.at(1.0001), Time.at(1))
-  rescue RuntimeError => e
-    assert_match(/should not provide an end time less than start time/, e.message)
-  end
-
-  def test_convert_to_microseconds
-    assert_equal((1_000_000_000), convert_to_microseconds(Time.at(1000)), 'time at 1000 seconds past epoch should be 1,000,000,000 usec')
-    assert_equal 1_000_000_000, convert_to_microseconds(1_000_000_000), 'should not mess with a number if passed in'
-    assert_raises(TypeError) do
-      convert_to_microseconds('whoo yeah buddy')
-    end
-  end
-
-  def test_convert_from_microseconds
-    assert_equal Time.at(1000), convert_from_microseconds(1_000_000_000), 'time at 1,000,000,000 usec should be 1000 seconds after epoch'
-    assert_equal Time.at(1000), convert_from_microseconds(Time.at(1000)), 'should not mess with a time passed in'
-    assert_raises(TypeError) do
-      convert_from_microseconds('10000000000')
-    end
-  end
-
-  def test_add_end_time_header
-    env = {}
-    start_time = Time.at(1)
-    add_end_time_header(start_time, env)
-    assert_equal({'HTTP_X_APPLICATION_START' => "t=#{convert_to_microseconds(Time.at(1))}"}, env, "should add the header to the env hash")
-  end
-
-  def test_parse_end_time_base
-    env = {}
-    env['HTTP_X_APPLICATION_START'] = "t=#{convert_to_microseconds(Time.at(1))}"
-    start_time = parse_end_time(env)
-    assert_equal(Time.at(1), start_time, "should pull the correct start time from the app header")
-  end
-
-  def test_get_matches_from_header
-    env = {'A HEADER' => 't=1000000'}
-    self.expects(:convert_from_microseconds).with(1000000).returns(Time.at(1))
-    matches = get_matches_from_header('A HEADER', env)
-    assert_equal [[nil, Time.at(1)]], matches, "should pull the correct time from the string"
-  end
-
-  def test_convert_to_name_time_pair
-    name = :foo
-    time = "1000000"
-
-    pair = convert_to_name_time_pair(name, time)
-    assert_equal [:foo, Time.at(1)], pair
-  end
-
-  def test_get_matches
-    str = "servera t=1000000, serverb t=1000000"
-    matches = get_matches(str) # start a fire
-    assert_equal [['servera', '1000000'], ['serverb', '1000000']], matches
-  end
-
-  def test_matches_with_bad_data
-    str = "stephan is a dumb lol"
-    matches = get_matches(str)
-    assert_equal [], matches
-
-    str = "t=100"
-    matches = get_matches(str)
-    assert_equal [[nil, '100']], matches
-
-    str = nil
-    matches = get_matches(str)
-    assert_equal [], matches
-  end
-  # each server should be one second, and the total would be 2 seconds
-  def test_record_individual_middleware_stats
-    matches = [['foo', Time.at(1000)], ['bar', Time.at(1001)]]
-    assert_calls_metrics('Middleware/foo', 'Middleware/bar') do
-      record_individual_middleware_stats(Time.at(1002), matches)
-    end
-    check_metric_time('Middleware/foo', 1.0, 0.1)
-    check_metric_time('Middleware/bar', 1.0, 0.1)
+  def assert_metric_value_in_delta(expected, metric_name, delta)
+    stats_engine = NewRelic::Agent.instance.stats_engine
+    stats_engine.clear_stats
+    yield
+    assert_in_delta(expected, stats_engine.get_stats(metric_name).total_call_time, delta)
   end
 end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: newrelic_rpm
 version: !ruby/object:Gem::Version
-  version: 3.5.6.55
-  prerelease: 
+  version: 3.5.7.57.beta
+  prerelease: 9
 platform: ruby
 authors:
 - Jason Clark
@@ -41,7 +41,7 @@ cert_chain:
   cHUySWFQWE92bTNUOEc0TzZxWnZobkxoL1VpZW4rK0RqOGVGQmVjVFBvTThw
   VmpLM3BoNQpuL0V3dVpDY0U2Z2h0Q0NNCi0tLS0tRU5EIENFUlRJRklDQVRF
   LS0tLS0K
-date: 2013-02-13 00:00:00.000000000 Z
+date: 2013-02-18 00:00:00.000000000 Z
 dependencies: []
 description: ! 'New Relic is a performance management system, developed by New Relic,
 
@@ -470,38 +470,13 @@ files:
 - lib/new_relic/build.rb
 homepage: http://www.github.com/newrelic/rpm
 licenses: []
-post_install_message: ! "\n# New Relic Ruby Agent Release Notes #\n\n## v3.5.6 ##\n\n
-  \ * Use HTTPS by default\n\n    The agent now defaults to using SSL when it communicates
-  with New Relic's\n    servers.  By defaults already configured, New Relic does not
-  transmit any\n    sensitive information (e.g. SQL parameters are masked), but SSL
-  adds \n    another layer of security.  Upgrading customers may need to remove the\n
-  \   \"ssl: false\" directive from their newrelic.yml to enable ssl.  Customers on\n
-  \   Jruby may need to install the jruby-openssl gem to take advantage of this\n
-  \   feature.\n\n  * Fix two Resque-related issues\n\n    Fixes a possible hang on
-  exit of an instrumented Resque master process\n    (https://github.com/defunkt/resque/issues/578),
-  as well as a file descriptor\n    leak that could occur during startup of the Resque
-  master process.\n\n  * Fix for error graph over 100%\n\n    Some errors were double
-  counted toward the overall error total. This\n    resulted in graphs with error
-  percentages over 100%. This duplication did\n    not impact the specific error traces
-  captured, only the total metric.\n\n  * Notice gracefully handled errors in Sinatra\n\n
-  \   When show_exceptions was set to false in Sinatra, errors weren't caught\n    by
-  New Relic's error collector. Now handled errors also have the chance\n    to get
-  reported back.\n\n  * Ruby 2.0 compatibility fixes\n\n    Ruby 2.0 no longer finds
-  protected methods by default, but will with a flag.\n    http://tenderlovemaking.com/2012/09/07/protected-methods-and-ruby-2-0.html\n\n
-  \   Thanks Ravil Bayramgalin and Charlie Somerville for the fixes.\n\n  * Auto-detect
-  Trinidad as dispatcher\n\n    Code already existing for detecting Trinidad as a
-  dispatcher, but was only\n    accessible via an ENV variable. This now auto-detects
-  on startup. Thanks\n    Robert Rasmussen for catching that.\n\n  * Coercion of types
-  in collector communication\n\n    Certain metrics can be recorded with a Ruby Rational
-  type, which JSON\n    serializes as a string rather than a floating point value.
-  We now treat\n    coerce each outgoing value, and log issues before sending the
-  data.\n\n  * Developer mode fix for chart error\n\n    Added require to fix a NameError
-  in developer mode for summary page. Thanks\n    to Ryan B. Harvey.\n\n  * Don't
-  touch deprecated RAILS_ROOT if on Rails 3\n\n    Under some odd startup conditions,
-  we would look for the RAILS_ROOT constant\n    after failing to find the ::Rails.root
-  in a Rails 3 app, causing deprecation\n    warnings. Thanks for Adrian Irving-Beer
-  for the fix.\n\nSee https://github.com/newrelic/rpm/blob/master/CHANGELOG for a
-  full list of\nchanges.\n"
+post_install_message: ! "\n# New Relic Ruby Agent Release Notes #\n\n## v3.5.7 ##\n\n
+  \ * Resolved some issues with tracking of frontend queue time, particularly\n    when
+  the agent is running on an app hosted on Heroku.  The agent will now\n    more reliably
+  parse the headers described in\n    https://newrelic.com/docs/features/tracking-front-end-time
+  and will\n    automatically detect whether the times provided are in seconds,\n
+  \   milliseconds or microseconds.\n\nSee https://github.com/newrelic/rpm/blob/master/CHANGELOG
+  for a full list of\nchanges.\n"
 rdoc_options:
 - --line-numbers
 - --inline-source