scout_apm 2.2.0.pre0 → 2.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1a02cb2156a16b3ddaa054eb34065653ed2abb7
-  data.tar.gz: 8023afd5daf499a2441ee54b9d7c5914ec00ac73
+  metadata.gz: 04b5ba56e4b5d547ab50ca2c7bf9de7ad51175b4
+  data.tar.gz: d8a85c252b959e001d5b92498de94ab844a76f6b
 SHA512:
-  metadata.gz: 2483b8a46a365e9568f36f7d2c46b8271b295b9c12c950694318d4dd09668d91d30a39cd3c0b7c03396ce07e9a5bd9f1f77710d9cc3175cfb09d57e41aea8d28
-  data.tar.gz: 979cae85d0a830fe3c4ff4be8640f7bc503e1bceb72bb096ae731d7679bf93b2d17de266dd300a57b23029014ccabc61d20863fcd6d8b104b3c82e4b632c91d5
+  metadata.gz: 49365dc262bd4065eb827c3affc362d02e82b25e1638d67912f2d32ed2ac75bd08c490cb6a5071d9d352ce0f0b9a0531d8adea657c39f42913f28291f961eed3
+  data.tar.gz: e931e86d96b7baf0a7fc329e17ce0683f953085b95cc6a01ca2ef67d13d1ba556d346a675fc96e9738ab65f5f2771fb90e49a2539952246f417873a3eebf16c1

data/CHANGELOG.markdown

@@ -2,6 +2,12 @@
 
 * ScoutProf BETA
 
+# 2.1.1
+
+* Fix an issue with AR instrumentation and complex queries
+* Fix use of configuration option `data_file`
+* Update unit tests
+
 # 2.1.0
 
 * Added ignore key to configuration to entirely ignore an endpoint. No traces

data/ext/stacks/extconf.rb

@@ -11,7 +11,11 @@ can_compile &&= have_func('rb_profile_frames')
 can_compile &&= have_func('rb_profile_frame_absolute_path')
 can_compile &&= have_func('rb_profile_frame_label')
 can_compile &&= have_func('rb_profile_frame_classpath')
-can_compile &&= have_library('rt') # for timer_create, timer_settime
+
+# Explicitly link against librt
+if have_macro('__linux__')
+  can_compile &&= have_library('rt') # for timer_create, timer_settime
+end
 
 # Pick the atomics implementation
 has_atomics_header = have_header("stdatomic.h")

data/ext/stacks/stacks.c

@@ -24,15 +24,15 @@
 #include <errno.h>
 #include <inttypes.h>
 #include <pthread.h>
-#include <semaphore.h>
-#include <setjmp.h>
 #include <signal.h>
 #include <stdbool.h>
 
 /*
  *  System
  */
+#ifdef __linux__
 #include <sys/syscall.h>
+#endif
 #include <sys/time.h>
 
 #include "scout_atomics.h"
@@ -57,12 +57,20 @@ VALUE interval;
 #define BUF_SIZE 512
 #define MAX_TRACES 2000
 
+#ifdef __linux__
 #define NANO_SECOND_MULTIPLIER  1000000  // 1 millisecond = 1,000,000 Nanoseconds
 const long INTERVAL = 1 * NANO_SECOND_MULTIPLIER; // milliseconds * NANO_SECOND_MULTIPLIER
-
 // For support of thread id in timer_create
 #define sigev_notify_thread_id _sigev_un._tid
 
+#else // __linux__
+
+const long INTERVAL = 1000; // 1ms
+
+#endif
+
+
+
 #ifdef RUBY_INTERNAL_EVENT_NEWOBJ
 
 // Forward Declarations
@@ -95,14 +103,17 @@ static __thread atomic_uint16_t _cur_traces_num = ATOMIC_INIT(0);
 static __thread atomic_uint32_t _skipped_in_gc = ATOMIC_INIT(0);
 static __thread atomic_uint32_t _skipped_in_signal_handler = ATOMIC_INIT(0);
 static __thread atomic_uint32_t _skipped_in_job_registered = ATOMIC_INIT(0);
+static __thread atomic_uint32_t _skipped_in_not_running = ATOMIC_INIT(0);
 
 static __thread VALUE _gc_hook;
+static __thread VALUE _ruby_thread;
 
 static __thread atomic_bool_t _job_registered = ATOMIC_INIT(false);
 
+#ifdef __linux__
 static __thread timer_t _timerid;
 static __thread struct sigevent _sev;
-
+#endif
 
 ////////////////////////////////////////////////////////////////////////////////////////
 // Global variables
@@ -116,7 +127,6 @@ scout_add_profiled_thread(pthread_t th)
   init_thread_vars();
   ATOMIC_STORE_BOOL(&_thread_registered, true);
 
-  fprintf(stderr, "APM DEBUG: Added thread id: %li\n", (unsigned long int)pthread_self());
   return 1;
 }
 
@@ -146,14 +156,14 @@ remove_profiled_thread(pthread_t th)
 
   ATOMIC_STORE_BOOL(&_ok_to_sample, false);
 
-  fprintf(stderr, "APM DEBUG: Would remove thread id: %li\n", (unsigned long int)th);
-
   // Unregister the _gc_hook from Ruby ObjectSpace, then free it as well as the _traces struct it wrapped.
   rb_gc_unregister_address(&_gc_hook);
   xfree(&_gc_hook);
   xfree(&_traces);
 
+#ifdef __linux__
   timer_delete(_timerid);
+#endif
 
   ATOMIC_STORE_BOOL(&_thread_registered, false);
   return 0;
@@ -196,6 +206,9 @@ rb_scout_uninstall_profiling(VALUE self)
 static VALUE
 rb_scout_install_profiling(VALUE self)
 {
+#ifndef __linux__
+  struct itimerval timer;
+#endif
   struct sigaction new_vtaction, old_vtaction;
 
   // We can only install once. If uninstall is called, we will NOT be able to call install again.
@@ -210,6 +223,13 @@ rb_scout_install_profiling(VALUE self)
   sigemptyset(&new_vtaction.sa_mask);
   sigaction(SIGALRM, &new_vtaction, &old_vtaction);
 
+#ifndef __linux__
+  timer.it_interval.tv_sec = 0;
+  timer.it_interval.tv_usec = INTERVAL; //FIX2INT(interval);
+  timer.it_value = timer.it_interval;
+  setitimer(ITIMER_REAL, &timer, 0);
+#endif
+
   rb_define_const(cStacks, "INSTALLED", Qtrue);
   scout_profiling_installed = 1;
 
@@ -260,6 +280,8 @@ init_thread_vars()
   ATOMIC_STORE_INT16(&_start_trace_index, 0);
   ATOMIC_STORE_INT16(&_cur_traces_num, 0);
 
+  _ruby_thread = rb_thread_current(); // Used as a check to avoid any Fiber switching silliness
+
   _traces = ALLOC_N(struct c_trace, MAX_TRACES); // TODO Check return
 
   _gc_hook = Data_Wrap_Struct(rb_cObject, &scoutprof_gc_mark, NULL, &_traces);
@@ -267,17 +289,19 @@ init_thread_vars()
 
   res = pthread_atfork(scout_parent_atfork_prepare, scout_parent_atfork_finish, NULL);
   if (res != 0) {
-    fprintf(stderr, "Pthread_atfork failed: %d\n", res);
+    fprintf(stderr, "APM-DEBUG: Pthread_atfork failed: %d\n", res);
   }
 
+#ifdef __linux__
   // Create timer to target this thread
   _sev.sigev_notify = SIGEV_THREAD_ID;
   _sev.sigev_signo = SIGALRM;
   _sev.sigev_notify_thread_id = syscall(SYS_gettid);
   _sev.sigev_value.sival_ptr = &_timerid;
   if (timer_create(CLOCK_MONOTONIC, &_sev, &_timerid) == -1) {
-    fprintf(stderr, "Time create failed: %d\n", errno);
+    fprintf(stderr, "APM-DEBUG: Time create failed: %d\n", errno);
   }
+#endif
 
   return;
 }
@@ -302,6 +326,8 @@ scout_profile_broadcast_signal_handler(int sig)
 
   if (rb_during_gc()) {
     ATOMIC_ADD(&_skipped_in_gc, 1);
+  } else if (rb_thread_current() != _ruby_thread) {
+    ATOMIC_ADD(&_skipped_in_not_running, 1);
   } else {
     if (ATOMIC_LOAD(&_job_registered) == false){
       register_result = rb_postponed_job_register(0, scout_record_sample, 0);
@@ -334,6 +360,10 @@ scout_record_sample()
     ATOMIC_ADD(&_skipped_in_gc, 1);
     return;
   }
+  if (rb_thread_current() != _ruby_thread) {
+    ATOMIC_ADD(&_skipped_in_not_running, 1);
+    return;
+  }
 
   cur_traces_num = ATOMIC_LOAD(&_cur_traces_num);
   start_frame_index = ATOMIC_LOAD(&_start_frame_index);
@@ -359,7 +389,7 @@ static VALUE rb_scout_profile_frames(VALUE self)
   VALUE traces, trace, trace_line;
 
   if (ATOMIC_LOAD(&_thread_registered) == false) {
-    fprintf(stderr, "Error: trying to get profiled frames on a non-profiled thread!\n");
+    fprintf(stderr, "APM-DEBUG: Error: trying to get profiled frames on a non-profiled thread!\n");
     ATOMIC_STORE_INT16(&_cur_traces_num, 0);
     return rb_ary_new();
   }
@@ -373,10 +403,14 @@ static VALUE rb_scout_profile_frames(VALUE self)
       if (_traces[i].num_tracelines > 0) {
         trace = rb_ary_new2(_traces[i].num_tracelines);
         for(n = 0; n < _traces[i].num_tracelines; n++) {
-          trace_line = rb_ary_new2(2);
-          rb_ary_store(trace_line, 0, _traces[i].frames_buf[n]);
-          rb_ary_store(trace_line, 1, INT2FIX(_traces[i].lines_buf[n]));
-          rb_ary_push(trace, trace_line);
+          if (TYPE(_traces[i].frames_buf[n]) == RUBY_T_DATA) { // We should always get valid frames from rb_profile_frames, but that doesn't always seem to be the case
+            trace_line = rb_ary_new2(2);
+            rb_ary_store(trace_line, 0, _traces[i].frames_buf[n]);
+            rb_ary_store(trace_line, 1, INT2FIX(_traces[i].lines_buf[n]));
+            rb_ary_push(trace, trace_line);
+          } else {
+            fprintf(stderr, "APM-DEBUG: Non-data frame is: 0x%04x\n", TYPE(_traces[i].frames_buf[n]));
+          }
         }
         rb_ary_push(traces, trace);
       }
@@ -397,15 +431,18 @@ static VALUE rb_scout_profile_frames(VALUE self)
 static void
 scout_start_thread_timer()
 {
+#ifdef __linux__
   struct itimerspec its;
   sigset_t mask;
+#endif
 
+#ifdef __linux__
   if (ATOMIC_LOAD(&_thread_registered) == false) return;
 
   sigemptyset(&mask);
   sigaddset(&mask, SIGALRM);
   if (sigprocmask(SIG_SETMASK, &mask, NULL) == -1) {
-    fprintf(stderr, "Block mask failed: %d\n", errno);
+    fprintf(stderr, "APM-DEBUG: Block mask failed: %d\n", errno);
   }
 
   its.it_value.tv_sec = 0;
@@ -414,25 +451,30 @@ scout_start_thread_timer()
   its.it_interval.tv_nsec = its.it_value.tv_nsec;
 
   if (timer_settime(_timerid, 0, &its, NULL) == -1) {
-    fprintf(stderr, "Timer set failed in start sampling: %d\n", errno);
+    fprintf(stderr, "APM-DEBUG: Timer set failed in start sampling: %d\n", errno);
   }
 
   if (sigprocmask(SIG_UNBLOCK, &mask, NULL) == -1) {
-    fprintf(stderr, "UNBlock mask failed: %d\n", errno);
+    fprintf(stderr, "APM-DEBUG: UNBlock mask failed: %d\n", errno);
   }
+#endif
 }
 
 static void
 scout_stop_thread_timer()
 {
+#ifdef __linux__
   struct itimerspec its;
+#endif
 
   if (ATOMIC_LOAD(&_thread_registered) == false) return;
 
+#ifdef __linux__
   memset((void*)&its, 0, sizeof(its));
   if (timer_settime(_timerid, 0, &its, NULL) == -1 ) {
-    fprintf(stderr, "Timer set failed: %d\n", errno);
+    fprintf(stderr, "APM-DEBUG: Timer set failed: %d\n", errno);
   }
+#endif
 }
 
 /* Per thread start sampling */
@@ -441,7 +483,9 @@ rb_scout_start_sampling(VALUE self)
 {
   scout_add_profiled_thread(pthread_self());
   ATOMIC_STORE_BOOL(&_ok_to_sample, true);
+#ifdef __linux__
   scout_start_thread_timer();
+#endif
   return Qtrue;
 }
 
@@ -450,7 +494,9 @@ static VALUE
 rb_scout_stop_sampling(VALUE self, VALUE reset)
 {
   if(ATOMIC_LOAD(&_ok_to_sample) == true ) {
+#ifdef __linux__
     scout_stop_thread_timer();
+#endif
   }
 
   ATOMIC_STORE_BOOL(&_ok_to_sample, false);
@@ -465,6 +511,7 @@ rb_scout_stop_sampling(VALUE self, VALUE reset)
     ATOMIC_STORE_INT32(&_skipped_in_gc, 0);
     ATOMIC_STORE_INT32(&_skipped_in_signal_handler, 0);
     ATOMIC_STORE_INT32(&_skipped_in_job_registered, 0);
+    ATOMIC_STORE_INT32(&_skipped_in_not_running, 0);
   }
   return Qtrue;
 }
@@ -520,6 +567,12 @@ rb_scout_skipped_in_job_registered(VALUE self)
   return INT2NUM(ATOMIC_LOAD(&_skipped_in_job_registered));
 }
 
+static VALUE
+rb_scout_skipped_in_not_running(VALUE self)
+{
+  return INT2NUM(ATOMIC_LOAD(&_skipped_in_not_running));
+}
+
 ////////////////////////////////////////////////////////////////
 // Fetch details from a frame
 ////////////////////////////////////////////////////////////////
@@ -587,6 +640,7 @@ void Init_stacks()
     rb_define_singleton_method(cStacks, "skipped_in_gc", rb_scout_skipped_in_gc, 0);
     rb_define_singleton_method(cStacks, "skipped_in_handler", rb_scout_skipped_in_handler, 0);
     rb_define_singleton_method(cStacks, "skipped_in_job_registered", rb_scout_skipped_in_job_registered, 0);
+    rb_define_singleton_method(cStacks, "skipped_in_not_running", rb_scout_skipped_in_not_running, 0);
 
     rb_define_const(cStacks, "ENABLED", Qtrue);
     rb_warning("Finished Initializing ScoutProf Native Extension");
@@ -679,6 +733,12 @@ rb_scout_skipped_in_job_registered(VALUE self)
   return INT2NUM(0);
 }
 
+static VALUE
+rb_scout_skipped_in_not_running(VALUE self)
+{
+  return INT2NUM(0);
+}
+
 static VALUE
 rb_scout_frame_klass(VALUE self, VALUE frame)
 {
@@ -735,6 +795,7 @@ void Init_stacks()
     rb_define_singleton_method(cStacks, "skipped_in_gc", rb_scout_skipped_in_gc, 0);
     rb_define_singleton_method(cStacks, "skipped_in_handler", rb_scout_skipped_in_handler, 0);
     rb_define_singleton_method(cStacks, "skipped_in_job_registered", rb_scout_skipped_in_job_registered, 0);
+    rb_define_singleton_method(cStacks, "skipped_in_not_running", rb_scout_skipped_in_not_running, 0);
 
     rb_define_const(cStacks, "ENABLED", Qfalse);
     rb_define_const(cStacks, "INSTALLED", Qfalse);

data/lib/scout_apm/agent.rb

@@ -55,7 +55,7 @@ module ScoutApm
       @request_histograms_by_time = Hash.new { |h, k| h[k] = ScoutApm::RequestHistograms.new }
 
       @store          = ScoutApm::Store.new
-      @layaway        = ScoutApm::Layaway.new
+      @layaway        = ScoutApm::Layaway.new(config, environment)
       @metric_lookup  = Hash.new
 
       @capacity       = ScoutApm::Capacity.new

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -57,12 +57,12 @@ module ScoutApm
         queue_layer = ScoutApm::Layer.new("Queue", queue)
         job_layer = ScoutApm::Layer.new("Job", job_class)
 
-        #if ScoutApm::Agent.instance.config.value('profile')
+        if ScoutApm::Agent.instance.config.value('profile') && SidekiqMiddleware.version_supports_profiling?
           # Capture ScoutProf if we can
           #req.enable_profiled_thread!
           #job_layer.set_root_class(job_class)
           #job_layer.traced!
-        #end
+        end
 
         req.start_layer(queue_layer)
         req.start_layer(job_layer)
@@ -77,6 +77,10 @@ module ScoutApm
         req.stop_layer # Job
         req.stop_layer # Queue
       end
-    end
+
+      def self.version_supports_profiling?
+        @@sidekiq_supports_profling ||= defined?(::Sidekiq::VERSION) && Gem::Dependency.new('', '~> 4.0').match?('', ::Sidekiq::VERSION.to_s)
+      end
+    end # SidekiqMiddleware
   end
 end

data/lib/scout_apm/config.rb

@@ -137,11 +137,17 @@ module ScoutApm
     end
 
     def initialize(overlays)
-      @overlays = overlays
+      @overlays = Array(overlays)
     end
 
     def value(key)
-      raw_value = @overlays.detect{ |overlay| overlay.has_key?(key) }.value(key)
+      o = @overlays.detect{ |overlay| overlay.has_key?(key) }
+      raw_value = if o
+                    o.value(key)
+                  else
+                    # No overlay said it could handle this key, bail out with nil.
+                    nil
+                  end
 
       coercion = SETTING_COERCIONS[key] || NullCoercion.new
       coercion.coerce(raw_value)

data/lib/scout_apm/instant/middleware.rb

@@ -61,7 +61,7 @@ module ScoutApm
                   :platform      => "ruby",
               }
               hash = ScoutApm::Serializers::PayloadSerializerToJson.rearrange_slow_transaction(trace)
-              hash.merge!(metadata:metadata)
+              hash.merge!(:metadata => metadata)
               payload = ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(hash)
 
               if env['HTTP_X_REQUESTED_WITH'] == 'XMLHttpRequest' || content_type.include?("application/json")
@@ -105,4 +105,4 @@ module ScoutApm
       end
     end
   end
-end
+end

data/lib/scout_apm/instruments/active_record.rb

@@ -173,7 +173,7 @@ module ScoutApm
         req.start_layer(layer)
         req.ignore_children!
         begin
-          find_by_sql_without_scout_instruments(*args)
+          find_by_sql_without_scout_instruments(*args, &block)
         ensure
           req.acknowledge_children!
           req.stop_layer
@@ -199,7 +199,7 @@ module ScoutApm
         req.start_layer(layer)
         req.ignore_children!
         begin
-          find_with_associations_without_scout_instruments(*args)
+          find_with_associations_without_scout_instruments(*args, &block)
         ensure
           req.acknowledge_children!
           req.stop_layer

data/lib/scout_apm/layaway.rb

@@ -18,8 +18,12 @@ module ScoutApm
     # Must be sortable as an integer
     TIME_FORMAT = "%Y%m%d%H%M"
 
-    def initialize(directory=nil)
-      @directory = directory
+    attr_reader :config
+    attr_reader :environment
+
+    def initialize(config, environment)
+      @config = config
+      @environment = environment
     end
 
     # Returns a Pathname object with the fully qualified directory where the layaway files can be placed.
@@ -30,12 +34,12 @@ module ScoutApm
     def directory
       return @directory if @directory
 
-      data_file = ScoutApm::Agent.instance.config.value("data_file")
-      data_file = File.dirname(data_file) if data_file && !File.directory?
+      data_file = config.value("data_file")
+      data_file = File.dirname(data_file) if data_file && !File.directory?(data_file)
 
       candidates = [
         data_file,
-        "#{ScoutApm::Agent.instance.environment.root}/tmp",
+        "#{environment.root}/tmp",
         "/tmp"
       ].compact
 

data/lib/scout_apm/layer.rb

@@ -154,6 +154,7 @@ module ScoutApm
           traces.skipped_in_gc = ScoutApm::Instruments::Stacks.skipped_in_gc
           traces.skipped_in_handler = ScoutApm::Instruments::Stacks.skipped_in_handler
           traces.skipped_in_job_registered = ScoutApm::Instruments::Stacks.skipped_in_job_registered
+          traces.skipped_in_not_running = ScoutApm::Instruments::Stacks.skipped_in_not_running
         end
       end
     end

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -139,7 +139,7 @@ module ScoutApm
             if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
               ScoutApm::Agent.instance.logger.debug do
                 traces_inspect = layer.traces.inspect
-                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}:\n#{traces_inspect}"
+                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
               end
             end
           else

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -133,7 +133,7 @@ module ScoutApm
             if layer.type =~ %r{^(Controller|Queue|Job)$}.freeze
               ScoutApm::Agent.instance.logger.debug do
                 traces_inspect = layer.traces.inspect
-                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}:\n#{traces_inspect}"
+                "****** Slow Request #{layer.type} Traces (#{layer.name}, tet: #{layer.total_exclusive_time}, tct: #{layer.total_call_time}), total raw traces: #{layer.traces.cube.total_count}, total clean traces: #{layer.traces.total_count}, skipped gc: #{layer.traces.skipped_in_gc}, skipped handler: #{layer.traces.skipped_in_handler}, skipped registered #{layer.traces.skipped_in_job_registered}, skipped not_running #{layer.traces.skipped_in_not_running}:\n#{traces_inspect}"
               end
             end
           else

data/lib/scout_apm/trace_compactor.rb

@@ -19,6 +19,7 @@ class TraceSet
   attr_accessor :skipped_in_gc
   attr_accessor :skipped_in_handler
   attr_accessor :skipped_in_job_registered
+  attr_accessor :skipped_in_not_running
 
   def initialize
     @raw_traces = []

data/lib/scout_apm/utils/fake_stacks.rb

@@ -77,6 +77,10 @@ module ScoutApm
         def skipped_in_job_registered(*args)
           0
         end
+
+        def skipped_in_not_running(*args)
+          0
+        end
       end
     end
   end

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.2.0.pre0"
+  VERSION = "2.2.0.pre1"
 end
 

data/lib/scout_apm.rb

@@ -128,7 +128,6 @@ require 'scout_apm/metric_meta'
 require 'scout_apm/metric_stats'
 require 'scout_apm/slow_transaction'
 require 'scout_apm/slow_job_record'
-require 'scout_apm/slow_item_set'
 require 'scout_apm/scored_item_set'
 require 'scout_apm/slow_request_policy'
 require 'scout_apm/slow_job_policy'

data/test/test_helper.rb

@@ -17,6 +17,34 @@ Kernel.module_eval do
   end
 end
 
+# A test helper class to create a temporary "configuration" we can control entirely purposes
+class FakeConfigOverlay
+  def initialize(values)
+    @values = values
+  end
+
+  def value(key)
+    @values[key]
+  end
+
+  def has_key?(key)
+    @values.has_key?(key)
+  end
+end
+
+class FakeEnvironment
+  def initialize(values)
+    @values = values
+  end
+
+  def method_missing(sym)
+    if @values.has_key?(sym)
+      @values[sym]
+    else
+      raise "#{sym} not found in FakeEnvironment"
+    end
+  end
+end
 
 # Helpers available to all tests
 class Minitest::Test
@@ -42,6 +70,16 @@ class Minitest::Test
     ScoutApm::Agent.instance.instance_variable_set("@logger", @logger)
   end
 
+  def make_fake_environment(values)
+    FakeEnvironment.new(values)
+  end
+
+  def make_fake_config(values)
+    ScoutApm::Config.new(FakeConfigOverlay.new(values))
+  end
+
   DATA_FILE_DIR = File.dirname(__FILE__) + '/tmp'
   DATA_FILE_PATH = "#{DATA_FILE_DIR}/scout_apm.db"
 end
+
+

data/test/unit/config_test.rb

@@ -4,7 +4,7 @@ require 'scout_apm/config'
 
 class ConfigTest < Minitest::Test
   def test_initalize_without_a_config
-    conf = ScoutApm::Config.new(nil)
+    conf = ScoutApm::Config.without_file
 
     # nil for random keys
     assert_nil conf.value("log_file_path")
@@ -21,7 +21,7 @@ class ConfigTest < Minitest::Test
     set_rack_env("production")
 
     conf_file = File.expand_path("../../data/config_test_1.yml", __FILE__)
-    conf = ScoutApm::Config.new(conf_file)
+    conf = ScoutApm::Config.with_file(conf_file)
 
     assert_equal "debug", conf.value('log_level')
     assert_equal "APM Test Conf (Production)", conf.value('name')

data/test/unit/layaway_test.rb

@@ -5,25 +5,20 @@ require 'scout_apm/metric_stats'
 require 'scout_apm/context'
 require 'scout_apm/store'
 
+require 'fileutils'
 class LayawayTest < Minitest::Test
-  def test_add_reporting_period
-    File.open(DATA_FILE_PATH, 'w') { |file| file.write(Marshal.dump(NEW_FORMAT)) }
-    ScoutApm::Agent.instance.start
+  def test_directory_uses_DATA_FILE_option
+    FileUtils.mkdir_p '/tmp/scout_apm_test/data_file_option'
+    config = make_fake_config("data_file" => "/tmp/scout_apm_test/data_file_option")
 
-    data = ScoutApm::Layaway.new
-    t = ScoutApm::StoreReportingPeriodTimestamp.new
-    data.add_reporting_period(t,ScoutApm::StoreReportingPeriod.new(t))
-    assert_equal [TIMESTAMP,t].sort_by(&:timestamp), Marshal.load(File.read(DATA_FILE_PATH)).keys.sort_by(&:timestamp)
+    assert_equal Pathname.new("/tmp/scout_apm_test/data_file_option"), ScoutApm::Layaway.new(config, ScoutApm::Agent.instance.environment).directory
   end
 
-  def test_merge_reporting_period
-    File.open(DATA_FILE_PATH, 'w') { |file| file.write(Marshal.dump(NEW_FORMAT)) }
-    layaway = ScoutApm::Layaway.new
-    layaway.add_reporting_period(TIMESTAMP, ScoutApm::StoreReportingPeriod.new(TIMESTAMP))
-    unmarshalled = Marshal.load(File.read(DATA_FILE_PATH))
-    assert_equal [TIMESTAMP], unmarshalled.keys
-  end
+  def test_directory_looks_for_root_slash_tmp
+    FileUtils.mkdir_p '/tmp/scout_apm_test/tmp'
+    config = make_fake_config({})
+    env = make_fake_environment(:root => "/tmp/scout_apm_test")
 
-  TIMESTAMP = ScoutApm::StoreReportingPeriodTimestamp.new(Time.parse("2015-01-01"))
-  NEW_FORMAT = {TIMESTAMP => ScoutApm::StoreReportingPeriod.new(TIMESTAMP)} # Format for 1.2+ agents
+    assert_equal Pathname.new("/tmp/scout_apm_test/tmp"), ScoutApm::Layaway.new(config, env).directory
+  end
 end

data/test/unit/serializers/payload_serializer_test.rb

@@ -6,26 +6,11 @@ require 'scout_apm/serializers/payload_serializer_to_json'
 require 'scout_apm/slow_transaction'
 require 'scout_apm/metric_meta'
 require 'scout_apm/metric_stats'
-require 'scout_apm/utils/fake_stack_prof'
 require 'scout_apm/context'
 require 'ostruct'
 require 'json' # to deserialize what has been manually serialized by the production code
 
-# stub the report_format value
-# class ScoutApm::Agent
-  # module Config
-    # def self.value(key)
-      # 'json'
-    # end
-  # end
-
-  # def self.instance
-    # OpenStruct.new(:config => Config)
-  # end
-# end
-
 class PayloadSerializerTest < Minitest::Test
-
   def test_serializes_metadata_as_json
     metadata = {
       :app_root => "/srv/app/rootz",
@@ -112,93 +97,6 @@ class PayloadSerializerTest < Minitest::Test
     assert_equal formatted_metrics, JSON.parse(payload)["metrics"]
   end
 
-  def test_serializes_slow_transactions_as_json
-    slow_transaction_metrics = {
-      ScoutApm::MetricMeta.new('ActiveRecord/all').tap { |meta|
-        meta.desc = "SELECT *\nfrom users where filter=?"
-        meta.extra = {:user => 'cooluser'}
-        meta.metric_id = nil
-        meta.scope = "Controller/apps/checkin"
-      } => ScoutApm::MetricStats.new.tap { |stats|
-        stats.call_count = 16
-        stats.max_call_time = 0.005338062
-        stats.min_call_time = 0.000613518
-        stats.sum_of_squares = 9.8040860751126e-05
-        stats.total_call_time = 0.033245704
-        stats.total_exclusive_time = 0.033245704
-      },
-      ScoutApm::MetricMeta.new("Controller/apps/checkin").tap { |meta|
-        meta.desc = nil
-        meta.extra = {}
-        meta.metric_id = nil
-        meta.scope = nil
-      } => ScoutApm::MetricStats.new.tap { |stats|
-        stats.call_count = 2
-        stats.max_call_time = 0.078521419
-        stats.min_call_time = 0.034881757
-        stats.sum_of_squares = 0.007382350213180609
-        stats.total_call_time = 0.113403176
-        stats.total_exclusive_time = 0.07813208899999999
-      }
-    }
-    context = ScoutApm::Context.new
-    context.add({"this" => "that"})
-    context.add_user({"hello" => "goodbye"})
-    slow_t = ScoutApm::SlowTransaction.new("http://example.com/blabla", "Buckethead/something/else", 1.23, slow_transaction_metrics, context, Time.at(1448198788), [], 10)
-    payload = ScoutApm::Serializers::PayloadSerializerToJson.serialize({}, {}, [slow_t], [], [])
-    formatted_slow_transactions = [
-      {
-        "key" => {
-          "bucket" => "Buckethead",
-          "name" => "something/else"
-        },
-        "time" => "2015-11-22 06:26:28 -0700",
-        "total_call_time" => 1.23,
-        "uri" => "http://example.com/blabla",
-        "context" => {"this"=>"that", "user"=>{"hello"=>"goodbye"}},
-        "prof" => [],
-        "score" => 10,
-        "metrics" => [
-          {
-            "key" => {
-              "bucket" => "ActiveRecord",
-              "name" => "all",
-              "desc" => "SELECT *\nfrom users where filter=?",
-              "extra" => {
-                "user" => "cooluser",
-              },
-              "scope" => {
-                "bucket" => "Controller",
-                "name" => "apps/checkin",
-              },
-            },
-            "call_count" => 16,
-            "max_call_time" => 0.005338062,
-            "min_call_time" => 0.000613518,
-            "total_call_time" => 0.033245704,
-            "total_exclusive_time" => 0.033245704,
-          },
-          {
-            "key" => {
-              "bucket" => "Controller",
-              "name" => "apps/checkin",
-              "desc" => nil,
-              "extra" => {},
-              "scope" => nil,
-            },
-            "call_count" => 2,
-            "max_call_time" => 0.078521419,
-            "min_call_time" => 0.034881757,
-            "total_call_time" => 0.113403176,
-            "total_exclusive_time" => 0.07813208899999999,
-          }
-        ]
-      }
-    ]
-
-    assert_equal formatted_slow_transactions, JSON.parse(payload)["slow_transactions"]
-  end
-
   def test_escapes_json_quotes
     metadata = {
       :quotie => "here are some \"quotes\"",

data/test/unit/slow_job_policy_test.rb

@@ -3,53 +3,4 @@ require 'test_helper'
 require 'scout_apm/slow_job_policy'
 
 class SlowJobPolicyTest < Minitest::Test
-  def test_first_call_is_not_slow
-    policy = ScoutApm::SlowJobPolicy.new
-    assert !policy.slow?("TestWorker", 10)
-  end
-
-  # All of these get faster and faster, so none are marked as slow.
-  def test_fast_calls_are_not_slow
-    policy = ScoutApm::SlowJobPolicy.new
-    assert !policy.slow?("TestWorker", 10)
-    assert !policy.slow?("TestWorker", 8)
-    assert !policy.slow?("TestWorker", 6)
-    assert !policy.slow?("TestWorker", 4)
-    assert !policy.slow?("TestWorker", 2)
-  end
-
-  def test_slow_calls_are_marked_as_slow
-    policy = ScoutApm::SlowJobPolicy.new
-    policy.slow?("TestWorker", 10) # Prime it with a not-slow
-
-    assert policy.slow?("TestWorker", 12)
-    assert policy.slow?("TestWorker", 14)
-    assert policy.slow?("TestWorker", 16)
-    assert policy.slow?("TestWorker", 18)
-  end
-
-  def test_mix_of_fast_and_slow
-    policy = ScoutApm::SlowJobPolicy.new
-    policy.slow?("TestWorker", 10) # Prime it with a not-slow
-
-    assert  policy.slow?("TestWorker", 12)
-    assert !policy.slow?("TestWorker", 8)
-    assert  policy.slow?("TestWorker", 13)
-    assert !policy.slow?("TestWorker", 6)
-  end
-
-  def test_different_workers_dont_interfere
-    policy = ScoutApm::SlowJobPolicy.new
-    policy.slow?("TestWorker", 10) # Prime it with a not-slow
-    policy.slow?("OtherWorker", 1.0) # Prime it with a not-slow
-
-    assert !policy.slow?("TestWorker", 8)
-    assert  policy.slow?("OtherWorker", 2)
-    assert !policy.slow?("TestWorker", 1)
-    assert  policy.slow?("OtherWorker", 3)
-    assert  policy.slow?("TestWorker", 12)
-    assert !policy.slow?("OtherWorker", 1)
-    assert  policy.slow?("TestWorker", 12)
-    assert  policy.slow?("OtherWorker", 4)
-  end
 end

data/test/unit/slow_request_policy_test.rb

@@ -32,11 +32,10 @@ class SlowRequestPolicyTest < Minitest::Test
 
     request.set_duration(10) # 10 seconds
     policy.last_seen[request.unique_name] = Time.now - 120 # 2 minutes since last seen
-    agent.request_histograms.add(request.unique_name, 1)
-
-    # Actual value I have in console is 1.599
-    assert policy.score(request) > 1.5
-    assert policy.score(request) < 2.0
+    ScoutApm::Agent.instance.request_histograms.add(request.unique_name, 1)
 
+    # Actual value I have in console is 1.499
+    assert policy.score(request) > 1.45
+    assert policy.score(request) < 1.55
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.2.0.pre0
+  version: 2.2.0.pre1
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-15 00:00:00.000000000 Z
+date: 2016-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rusage
@@ -204,7 +204,6 @@ files:
 - lib/scout_apm/server_integrations/thin.rb
 - lib/scout_apm/server_integrations/unicorn.rb
 - lib/scout_apm/server_integrations/webrick.rb
-- lib/scout_apm/slow_item_set.rb
 - lib/scout_apm/slow_job_policy.rb
 - lib/scout_apm/slow_job_record.rb
 - lib/scout_apm/slow_request_policy.rb
@@ -239,7 +238,6 @@ files:
 - test/unit/metric_set_test.rb
 - test/unit/scored_item_set_test.rb
 - test/unit/serializers/payload_serializer_test.rb
-- test/unit/slow_item_set_test.rb
 - test/unit/slow_job_policy_test.rb
 - test/unit/slow_request_policy_test.rb
 - test/unit/sql_sanitizer_test.rb

data/lib/scout_apm/slow_item_set.rb

@@ -1,80 +0,0 @@
-# In order to keep load down, only record a sample of Slow Items (Transactions
-# or Jobs).  In order to make that sampling as fair as possible, follow a basic
-# algorithm:
-#
-# When adding a new Slow Item:
-#  * Just add it if there is an open spot
-#  * If there isn't an open spot, attempt to remove an over-represented
-#    item instead ("attempt_to_evict"). Overrepresented is simply "has more
-#    than @fair number of Matching Items in the set". The fastest of the
-#    overrepresented items is removed.
-#  * If there isn't an open spot, and no Item is valid to evict, drop the
-#    incoming Item without adding.
-#
-# There is no way to remove Items from this set, create a new object
-# for each reporting period.
-#
-# Item must respond to:
-#   #metric_name - string - grouping key to see if one kind of thing is overrepresented
-#   #total_call_time - float - duration of the item
-
-module ScoutApm
-  class SlowItemSet
-    include Enumerable
-
-    DEFAULT_TOTAL = 10
-    DEFAULT_FAIR = 1
-
-    attr_reader :total
-    attr_reader :fair
-
-    def initialize(total=DEFAULT_TOTAL, fair=DEFAULT_FAIR)
-      @total = total
-      @fair = fair
-      @items = []
-    end
-
-    def each
-      @items.each { |s| yield s }
-    end
-
-    def <<(item)
-      return if attempt_append(item)
-      attempt_to_evict
-      attempt_append(item)
-    end
-
-    def empty_slot?
-      @items.length < total
-    end
-
-    def attempt_append(item)
-      if empty_slot?
-        @items.push(item)
-        true
-      else
-        false
-      end
-    end
-
-    def attempt_to_evict
-      return if @items.length == 0
-
-      overrepresented = @items.
-        group_by { |item| unique_name_for(item) }.
-        to_a.
-        sort_by { |(_, items)| items.length }.
-        last
-
-      if overrepresented[1].length > fair
-        fastest = overrepresented[1].sort_by { |item| item.total_call_time }.first
-        @items.delete(fastest)
-      end
-    end
-
-    # Determine this items' "hash key"
-    def unique_name_for(item)
-      item.metric_name
-    end
-  end
-end

data/test/unit/slow_item_set_test.rb

@@ -1,94 +0,0 @@
-require 'test_helper'
-
-require 'scout_apm/slow_item_set'
-require 'scout_apm/slow_transaction'
-
-class SlowItemSetTest < Minitest::Test
-  def test_adding_to_empty_set
-    set = ScoutApm::SlowItemSet.new(3, 1)
-    set << make_slow("Controller/Foo")
-    assert_equal 1, set.count
-  end
-
-  def test_adding_to_partially_full_set
-    set = ScoutApm::SlowItemSet.new(3, 1)
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    assert_equal 2, set.count
-  end
-
-  def test_overflow_of_one_type
-    max_size = 3
-    set = ScoutApm::SlowItemSet.new(max_size, 1)
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    assert_equal max_size, set.count
-  end
-
-  def test_eviction_of_overrepresented
-    max_size = 3
-    set = ScoutApm::SlowItemSet.new(max_size, 1)
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Bar")
-
-    # 3 total
-    assert_equal max_size, set.count
-    assert_equal 1, set.select{|sl| sl.metric_name == "Controller/Bar"}.length
-    assert_equal 2, set.select{|sl| sl.metric_name == "Controller/Foo"}.length
-  end
-
-  # Fill the set with /Foo records, then add a /Bar to evict. Check that the
-  # evicted one was the fastest of the Foos
-  def test_eviction_of_fastest
-    max_size = 3
-    set = ScoutApm::SlowItemSet.new(max_size, 1)
-
-    [1,2,3].shuffle.each do |seconds| # Shuffle to remove any assumptions on order
-      set << make_slow("Controller/Foo", seconds)
-    end
-    set << make_slow("Controller/Bar", 8)
-
-    # The foo taking 1 second should be evicted
-    assert_equal 2, set.select{|sl| sl.metric_name == "Controller/Foo"}.map{ |sl| sl.total_call_time}.min
-  end
-
-  def test_eviction_when_no_overrepresented
-    max_size = 4
-    fair = 2
-    set = ScoutApm::SlowItemSet.new(max_size, fair)
-
-    # Full, but each is at fair level
-    set << make_slow("Controller/Bar")
-    set << make_slow("Controller/Bar")
-    set << make_slow("Controller/Foo")
-    set << make_slow("Controller/Foo")
-
-    set << make_slow("Controller/Quux")
-    assert_equal max_size, set.count
-    assert_equal 0, set.select{|sl| sl.metric_name == "Controller/Quux" }.length
-  end
-
-  ##############
-  #### Helpers
-  ##############
-
-  def make_slow(metric, time=5)
-    ScoutApm::SlowTransaction.new(
-      "http://foo.app/#{metric}",
-      metric,
-      time,
-      {}, # metrics
-      {}, # context
-      Time.now, # end time
-      [], # stackprof
-      0)
-  end
-end