scout_rails 1.1.5.pre → 1.1.5.pre3

data/CHANGELOG.markdown

@@ -1,3 +1,8 @@
+# 1.1.5
+
+* sending array of transaction samples
+* sending transaction samples across processes
+
 # 1.1.3
 
 * Rails 4 support

data/lib/scout_rails/agent/reporting.rb

@@ -7,8 +7,10 @@ module ScoutRails
       def process_metrics
         logger.debug "Processing metrics"
         run_samplers
-        metrics = layaway.deposit_and_deliver
-        if metrics.any?
+        payload = layaway.deposit_and_deliver
+        metrics = payload[:metrics]
+        samples = payload[:samples]
+        if payload.any?
           add_metric_ids(metrics)  
           logger.warn "Some data may be lost - metric size is at limit" if metrics.size == ScoutRails::Store::MAX_SIZE
           # for debugging, count the total number of requests    
@@ -18,7 +20,7 @@ module ScoutRails
               controller_count += stats.call_count
             end
           end      
-          sample, samples = store.fetch_and_reset_samples!  
+          sample = store.fetch_and_reset_sample!  
           payload = Marshal.dump(:metrics => metrics, :sample => sample, :samples => samples)
           STATSD.timing('scout_rails.payload_kb',payload.size/1024)
           STATSD.timing('scout_rails.samples',samples.size)

data/lib/scout_rails/layaway.rb

@@ -2,8 +2,8 @@
 # 1. A centralized store for multiple Agent processes. This way, only 1 checkin is sent to Scout rather than 1 per-process.
 # 2. Bundling up reports from multiple timeslices to make updates more efficent server-side.
 # 
-# Metrics are stored in a Hash, where the keys are Time.to_i on the minute. When depositing data, 
-# metrics are either merged with an existing time or placed in a new key.
+# Data is stored in a Hash, where the keys are Time.to_i on the minute. The value is a Hash {:metrics => Hash, :samples => Array}.
+# When depositing data, the new data is either merged with an existing time or placed in a new key.
 class ScoutRails::Layaway
   attr_accessor :file
   def initialize
@@ -11,24 +11,35 @@ class ScoutRails::Layaway
   end
   
   def deposit_and_deliver
-    new_data = ScoutRails::Agent.instance.store.metric_hash
-    log_deposited_requests(new_data)
+    new_metrics = ScoutRails::Agent.instance.store.metric_hash
+    log_deposited_metrics(new_metrics)
+    log_deposited_samples(ScoutRails::Agent.instance.store.samples)
     to_deliver = {}
     file.read_and_write do |old_data|
       old_data ||= Hash.new
       # merge data
       # if (1) there's data in the file and (2) there isn't any data yet for the current minute, this means we've 
       # collected all metrics for the previous slots and we're ready to deliver.
+      #
+      # Example w/2 processes:
+      #
+      # 12:00:34 ---
+      # Process 1: old_data.any? => false, so deposits.
+      # Process 2: old_data_any? => true and old_data[12:00].nil? => false, so deposits. 
+      #
+      # 12:01:34 ---
+      # Process 1: old_data.any? => true and old_data[12:01].nil? => true, so delivers metrics.
+      # Process 2: old_data.any? => true and old_data[12:01].nil? => false, so deposits.
       if old_data.any? and old_data[slot].nil?
         to_deliver = old_data
         old_data = Hash.new
       elsif old_data.any?
-        ScoutRails::Agent.instance.logger.debug "Not yet time to deliver metrics for slot [#{Time.at(old_data.keys.sort.last).strftime("%m/%d/%y %H:%M:%S %z")}]"
+        ScoutRails::Agent.instance.logger.debug "Not yet time to deliver payload for slot [#{Time.at(old_data.keys.sort.last).strftime("%m/%d/%y %H:%M:%S %z")}]"
       else
         ScoutRails::Agent.instance.logger.debug "There is no data in the layaway file to deliver."
       end
-      old_data[slot]=ScoutRails::Agent.instance.store.merge_data_and_clear(old_data[slot] || Hash.new)
-      log_saved_requests(old_data,new_data)
+      old_data[slot]=ScoutRails::Agent.instance.store.merge_data_and_clear(old_data[slot] || {:metrics => {}, :samples => []})
+      log_saved_data(old_data,new_metrics)
       old_data
     end
     to_deliver.any? ? validate_data(to_deliver) : {}
@@ -53,32 +64,37 @@ class ScoutRails::Layaway
     ScoutRails::Agent.instance.logger.debug $!.backtrace
   end
   
+  # Data is stored under timestamp-keys (without the second).
   def slot
     t = Time.now
     t -= t.sec
     t.to_i
   end
   
-  def log_deposited_requests(new_data)
+  def log_deposited_metrics(new_metrics)
     controller_count = 0
-    new_data.each do |meta,stats|
+    new_metrics.each do |meta,stats|
       if meta.metric_name =~ /\AController/
         controller_count += stats.call_count
       end
     end
     ScoutRails::Agent.instance.logger.debug "Depositing #{controller_count} requests into #{Time.at(slot).strftime("%m/%d/%y %H:%M:%S %z")} slot."
   end
+
+  def log_deposited_samples(new_samples)
+    ScoutRails::Agent.instance.logger.debug "Depositing #{new_samples.size} samples into #{Time.at(slot).strftime("%m/%d/%y %H:%M:%S %z")} slot."
+  end
   
-  def log_saved_requests(old_data,new_data)
+  def log_saved_data(old_data,new_metrics)
     ScoutRails::Agent.instance.logger.debug "Saving the following #{old_data.size} time slots locally:"
     old_data.each do |k,v|
       controller_count = 0
-      new_data.each do |meta,stats|
+      new_metrics.each do |meta,stats|
         if meta.metric_name =~ /\AController/
           controller_count += stats.call_count
         end
       end
-      ScoutRails::Agent.instance.logger.debug "#{Time.at(k).strftime("%m/%d/%y %H:%M:%S %z")} => #{controller_count} requests"
+      ScoutRails::Agent.instance.logger.debug "#{Time.at(k).strftime("%m/%d/%y %H:%M:%S %z")} => #{controller_count} requests and #{v[:samples].size} samples"
     end
   end
 end

data/lib/scout_rails/store.rb

@@ -85,7 +85,7 @@ class ScoutRails::Store
       duplicate.each_pair do |k,v|
         duplicate[k.dup] = v.dup
       end  
-      merge_data(duplicate.merge({meta.dup => stat.dup})) # aggregrates + controller 
+      merge_metrics(duplicate.merge({meta.dup => stat.dup})) # aggregrates + controller 
     end
   end
   
@@ -137,19 +137,18 @@ class ScoutRails::Store
       # tree map of all slow transactions
       if parent_stat.total_call_time >= 2
         @samples.push(ScoutRails::TransactionSample.new(uri,parent_meta.metric_name,parent_stat.total_call_time,transaction_hash.dup))
+        ScoutRails::Agent.instance.logger.debug "Slow transaction sample added. Array Size: #{@samples.size}"
       end
     end
   end
 
-  # Returns the slow samples and resets the values - used when reporting. 
-  def fetch_and_reset_samples!
+  # Returns the slow sample and resets the values - used when reporting. 
+  def fetch_and_reset_sample!
     sample = @sample
-    samples = @samples
     @transaction_sample_lock.synchronize do
       self.sample = nil
-      self.samples = Array.new
     end
-    [sample,samples]
+    sample
   end
   
   # Finds or creates the metric w/the given name in the metric_hash, and updates the time. Primarily used to 
@@ -168,14 +167,7 @@ class ScoutRails::Store
   
   # Combines old and current data
   def merge_data(old_data)
-    old_data.each do |old_meta,old_stats|
-      if stats = metric_hash[old_meta]
-        metric_hash[old_meta] = stats.combine!(old_stats)
-      elsif metric_hash.size < MAX_SIZE
-        metric_hash[old_meta] = old_stats
-      end
-    end
-    metric_hash
+    {:metrics => merge_metrics(old_data[:metrics]), :samples => merge_samples(old_data[:samples])}
   end
   
   # Merges old and current data, clears the current in-memory metric hash, and returns
@@ -183,6 +175,25 @@ class ScoutRails::Store
   def merge_data_and_clear(old_data)
     merged = merge_data(old_data)
     self.metric_hash =  {}
+    # TODO - is this lock needed?
+    @transaction_sample_lock.synchronize do
+      self.samples = []
+    end
     merged
   end
+
+  def merge_metrics(old_metrics)
+    old_metrics.each do |old_meta,old_stats|
+      if stats = metric_hash[old_meta]
+        metric_hash[old_meta] = stats.combine!(old_stats)
+      elsif metric_hash.size < MAX_SIZE
+        metric_hash[old_meta] = old_stats
+      end
+    end
+    metric_hash
+  end
+
+  def merge_samples(old_samples)
+    self.samples += old_samples
+  end
 end # class Store

data/lib/scout_rails/version.rb

@@ -1,3 +1,3 @@
 module ScoutRails
-  VERSION = "1.1.5.pre"
+  VERSION = "1.1.5.pre3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_rails
 version: !ruby/object:Gem::Version
-  version: 1.1.5.pre
+  version: 1.1.5.pre3
   prerelease: 6
 platform: ruby
 authors: