scout_apm 1.6.1 → 1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cf9f6ca2e9039c82d55f3a6a120d3a9bb6b821d8
-  data.tar.gz: f7530422c05a1437175806886a4d2e1873e4c59d
+  metadata.gz: 5dd32dd69f13bbb9d647caa56a2e17cfe89340e2
+  data.tar.gz: 4663534020a6d20226f566abd8ce9e3ba90d9263
 SHA512:
-  metadata.gz: 5fd9a17e01253f7c2f1f1dea6933c94ea856411755e375442ab9dc85201f0618695c6aae72a3180f0d87e73827365a5b34e9b0f11591f1acd1965780346b708d
-  data.tar.gz: 2f49baa4dff8fab01ad7244b3298ea06ff424d24c682dd2dcd50ed53f6bc9231e16b7b29f9764c6e0658330fc630da0c0c537ec3bb5653c1fe52bc46555b2eba
+  metadata.gz: 595e782a27c8721ba8fc7dc4760b672b94f37aed634b1ed68eced23d4d72401052dd2792b5889534cd287cecfb9fdcfbffdbf081fab38876b40c477346f5816d
+  data.tar.gz: 562921bd5380679a1efbf24dedced0becd8ab2da347897550b09468f6c967e9eada8e5702954d52c7dafcdbfd666ae79b1056c574656c61cbc8d269e6cca4293

data/.gitignore

@@ -7,3 +7,4 @@ pkg/*
 .idea
 test/tmp/*coverage/*
 coverage/*
+lib/*.bundle

data/CHANGELOG.markdown

@@ -1,3 +1,8 @@
+# 1.6.2
+
+* Use a more flexible approach to storing "Layaway Files" (the temporary data
+  files the agent uses).
+
 # 1.6.1
 
 * Remove old & unused references to Stackprof. Prevent interaction with intentional usage of Stackprof

data/LICENSE.md

@@ -0,0 +1,9 @@
+All components of this product, except where noted, are Copyright (c) 2013-2016 Zimuth, Inc DBA Scout. All rights reserved.
+
+Certain inventions disclosed in this file may be claimed within patents owned or patent applications filed by Zimuth, Inc. or third parties.
+
+Subject to the terms of this notice, Zimuth grants you a nonexclusive, nontransferable license, without the right to sublicense, to (a) install and execute one copy of these files on any number of workstations owned or controlled by you and (b) distribute verbatim copies of these files to third parties. As a condition to the foregoing grant, you must provide this notice along with each copy you distribute and you must not remove, alter, or obscure this notice. All other use, reproduction, modification, distribution, or other exploitation of these files is strictly prohibited, except as may be set forth in a separate written license agreement between you and Zimuth. The terms of any such license agreement will control over this notice. The license stated above will be automatically terminated and revoked if you exceed its scope or violate any of the terms of this notice.
+
+This License does not grant permission to use the trade names, trademarks, service marks, or product names of Zimuth, except as required for reasonable and customary use in describing the origin of this file and reproducing the content of this notice. You may not mark or brand this file with any trade name, trademarks, service marks, or product names other than the original brand (if any) provided by Zimuth.
+
+Unless otherwise expressly agreed by Zimuth in a separate written license agreement, these files are provided AS IS, WITHOUT WARRANTY OF ANY KIND, including without any implied warranties of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE, or NON-INFRINGEMENT. As a condition to your use of these files, you are solely responsible for such use. Zimuth will have no liability to you for direct, indirect, consequential, incidental, special, or punitive damages or for lost profits or data.

data/lib/scout_apm/agent/reporting.rb

@@ -24,17 +24,35 @@ module ScoutApm
         report_to_server
       end
 
-      MAX_AGE_TO_REPORT = (10 * 60) # ten minutes as seconds
-
       # In a running app, one process will get one period ready for delivery, the others will see 0.
       def report_to_server
-        reporting_periods = layaway.periods_ready_for_delivery
-        reporting_periods.reject! {|rp| rp.timestamp.age_in_seconds > MAX_AGE_TO_REPORT }
-        reporting_periods.each do |rp|
-          deliver_period(rp)
+        period_to_report = ScoutApm::StoreReportingPeriodTimestamp.minutes_ago(2)
+
+        logger.debug("Attempting to claim #{period_to_report.to_s}")
+
+        did_write = layaway.with_claim(period_to_report) do |rps|
+          logger.debug("Succeeded claiming #{period_to_report.to_s}")
+
+          begin
+            merged = rps.inject { |memo, rp| memo.merge(rp) }
+            logger.debug("Merged #{rps.length} reporting periods, delivering")
+            deliver_period(merged)
+            true
+          rescue => e
+            logger.debug("Error merging reporting periods #{e.message}")
+            logger.debug("Error merging reporting periods #{e.backtrace}")
+            false
+          end
+
+        end
+
+        if !did_write
+          logger.debug("Failed to obtain claim for #{period_to_report.to_s}")
         end
+
       end
 
+
       def deliver_period(reporting_period)
         metrics = reporting_period.metrics_payload
         slow_transactions = reporting_period.slow_transactions_payload

data/lib/scout_apm/layaway.rb

@@ -1,53 +1,156 @@
-# Stores StoreReportingPeriod objects in a file before sending them to the server.
-# 1. A centralized store for multiple Agent processes. This way, only 1 checkin is sent to Scout rather than 1 per-process.
+# Stores StoreReportingPeriod objects in a per-process file before sending them to the server.
+# Coordinates a single process to collect up all individual files, merge them, then send.
+#
+# Each layaway file is named basedir/scout_#{timestamp}_#{pid}.data
+#   Where timestamp is in the format:
+#   And PID is the process id of the running process
+#
 module ScoutApm
   class Layaway
-    attr_accessor :file
+    # How old a file needs to be in Seconds before it gets reported.
+    REPORTING_AGE = 120
 
-    def initialize
-      @file = ScoutApm::LayawayFile.new
+    # How long to let a stale file sit before deleting it.
+    # Letting it sit a bit may be useful for debugging
+    STALE_AGE = 10 * 60
+
+    # A strftime format string for how we render timestamps in filenames.
+    # Must be sortable as an integer
+    TIME_FORMAT = "%Y%m%d%H%M"
+
+    def initialize(directory=nil)
+      @directory = directory
+    end
+
+    # Returns a Pathname object with the fully qualified directory where the layaway files can be placed.
+    # That directory must be writable by this process.
+    #
+    # Don't set this in initializer, since it relies on agent instance existing to figure out the value.
+    #
+    def directory
+      return @directory if @directory
+
+      data_file = ScoutApm::Agent.instance.config.value("data_file")
+      data_file = File.dirname(data_file) if data_file && !File.directory?
+
+      candidates = [
+        data_file,
+        "#{ScoutApm::Agent.instance.environment.root}/tmp",
+        "/tmp"
+      ].compact
+
+      found = candidates.detect { |dir| File.writable?(dir) }
+      ScoutApm::Agent.instance.logger.debug("Storing Layaway Files in #{found}")
+      @directory = Pathname.new(found)
+    end
+
+    def write_reporting_period(reporting_period)
+      filename = file_for(reporting_period.timestamp)
+      layaway_file = LayawayFile.new(filename)
+      layaway_file.write(reporting_period)
     end
 
-    def add_reporting_period(time, reporting_period)
-      file.read_and_write do |existing_data|
-        existing_data ||= Hash.new
-        ScoutApm::Agent.instance.logger.debug("AddReportingPeriod: Adding a reporting_period with timestamp: #{reporting_period.timestamp.to_s}, and #{reporting_period.request_count} requests")
+    # Claims a given timestamp (getting a lock on a particular filename),
+    # then yields ReportingPeriods collected up from all the files.
+    # If the yield returns truthy, delete the layaway files that made it up.
+    def with_claim(timestamp)
+      coordinator_file = glob_pattern(timestamp, :coordinator)
+
+
+      # This file gets deleted only by a process that successfully obtained a lock
+      f = File.open(coordinator_file, File::RDWR | File::CREAT)
+      begin
+        # Nonblocking, Exclusive lock.
+        if f.flock(File::LOCK_EX | File::LOCK_NB)
+
+          ScoutApm::Agent.instance.logger.debug("Obtained Reporting Lock")
 
-        existing_data = existing_data.merge(time => reporting_period) {|key, old_val, new_val|
-          old_req = old_val.request_count
-          new_req = new_val.request_count
-          ScoutApm::Agent.instance.logger.debug("Merging Two reporting periods (#{old_val.timestamp.to_s}, #{new_val.timestamp.to_s}): old req #{old_req}, new req #{new_req}")
+          files = all_files_for(timestamp).reject{|l| l.to_s == coordinator_file.to_s }
+          rps = files.map{ |layaway| LayawayFile.new(layaway).load }.compact
+          if rps.any?
+            yield rps
 
-          old_val.merge(new_val)
-        }
+            delete_files_for(timestamp) # also removes the coodinator_file
+            delete_stale_files(timestamp.to_time - STALE_AGE)
+          else
+            File.unlink(coordinator_file)
+            ScoutApm::Agent.instance.logger.debug("No layaway files to report")
+          end
 
-        ScoutApm::Agent.instance.logger.debug("AddReportingPeriod: AfterMerge Timestamps: #{existing_data.keys.map(&:to_s).inspect}")
-        existing_data
+          # Unlock the file when done!
+          f.flock(File::LOCK_UN | File::LOCK_NB)
+          f.close
+          true
+        else
+          # Didn't obtain lock, another process is reporting. Return false from this function, but otherwise no work
+          f.close
+          false
+        end
       end
     end
 
-    REPORTING_INTERVAL = 60 # seconds
+    def delete_files_for(timestamp)
+      all_files_for(timestamp).each { |layaway| File.unlink(layaway) }
+    end
+
+    def delete_stale_files(older_than)
+      all_files_for(:all).
+        map { |filename| timestamp_from_filename(filename) }.
+        compact.
+        uniq.
+        select { |timestamp| timestamp.to_i < older_than.strftime(TIME_FORMAT).to_i }.
+          tap  { |timestamps| ScoutApm::Agent.instance.logger.debug("Deleting stale layaway files with timestamps: #{timestamps.inspect}") }.
+        map    { |timestamp| delete_files_for(timestamp) }
+    end
 
-    # Returns an array of ReportingPeriod objects that are ready to be pushed to the server
-    def periods_ready_for_delivery
-      ready_for_delivery = []
-      file.read_and_write do |existing_data|
-        existing_data ||= {}
+    private
+
+    ##########################################
+    # Looking up files
+
+    def file_for(timestamp)
+      glob_pattern(timestamp)
+    end
 
-        ScoutApm::Agent.instance.logger.debug("PeriodsReadyForDeliver: All Timestamps: #{existing_data.keys.map(&:to_s).inspect}")
+    def all_files_for(timestamp)
+      Dir[glob_pattern(timestamp, :all)]
+    end
 
-        ready_for_delivery = existing_data.to_a.select {|time, rp| should_send?(rp) } # Select off the values we want. to_a is needed for compatibility with Ruby 1.8.7.
+    # Timestamp should be either :all or a Time-ish object that responds to strftime (StoreReportingPeriodTimestamp does)
+    # if timestamp == :all then find all timestamps, otherwise format it.
+    # if pid == :all, get the files for all
+    def glob_pattern(timestamp, pid=$$)
+      timestamp_pattern = format_timestamp(timestamp)
+      pid_pattern = format_pid(pid)
+      directory + "scout_#{timestamp_pattern}_#{pid_pattern}.data"
+    end
 
-        # Rewrite anything not plucked out back to the file
-        existing_data.reject {|k, v| ready_for_delivery.map(&:first).include?(k) }
+    def format_timestamp(timestamp)
+      if timestamp == :all
+        "*"
+      elsif timestamp.respond_to?(:strftime)
+        timestamp.strftime(TIME_FORMAT)
+      else
+        timestamp.to_s
       end
+    end
 
-      return ready_for_delivery.map(&:last)
+    def format_pid(pid)
+      if pid == :all
+        "*"
+      else
+        pid.to_s
+      end
     end
 
-    # We just want to send anything older than X
-    def should_send?(reporting_period)
-      reporting_period.timestamp.age_in_seconds > (REPORTING_INTERVAL * 2)
+    def timestamp_from_filename(filename)
+      match = filename.match(%r{scout_(.*)_.*\.data})
+      if match
+        match[1]
+      else
+        nil
+      end
     end
   end
 end
+

data/lib/scout_apm/layaway_file.rb

@@ -1,94 +1,36 @@
-# Logic for the serialized file access
+# A single layaway file.  See Layaway for the management of the group of files.
 module ScoutApm
   class LayawayFile
-    def path
-      return @path if @path
+    attr_reader :path
 
-      candidates = [
-        ScoutApm::Agent.instance.config.value("data_file"),
-        "#{ScoutApm::Agent.instance.default_log_path}/scout_apm.db",
-        "#{ScoutApm::Agent.instance.environment.root}/tmp/scout_apm.db"
-      ]
-
-      candidates.each do |candidate|
-        next if candidate.nil?
-
-        begin
-          ScoutApm::Agent.instance.logger.debug("Checking Layaway File Location: #{candidate}")
-          File.open(candidate, "w") { |f| } # Open & Close to check that we can
-
-          # No exception, it is valid
-          ScoutApm::Agent.instance.logger.info("Layaway File location found: #{candidate}")
-          @path = candidate
-          return @path
-        rescue Exception
-          ScoutApm::Agent.instance.logger.debug("Couldn't open layaway file for test write at #{candidate}")
-        end
-      end
-
-      ScoutApm::Agent.instance.logger.error("No valid layaway file found, please set a location in the configuration key `data_file`")
-      nil
+    def initialize(path)
+      @path = path
     end
 
-    def dump(object)
-      Marshal.dump(object)
-    end
-
-    def load(dump)
-      if dump.size == 0
-        ScoutApm::Agent.instance.logger.debug("No data in layaway file.")
-        return nil
-      end
-      Marshal.load(dump)
+    def load
+      data = File.open(path, "r") { |f| read_raw(f) }
+      deserialize(data)
     rescue NameError, ArgumentError, TypeError => e
+      # Marshal error
       ScoutApm::Agent.instance.logger.info("Unable to load data from Layaway file, resetting.")
       ScoutApm::Agent.instance.logger.debug("#{e.message}, #{e.backtrace.join("\n\t")}")
       nil
     end
 
-    def read_and_write
-      File.open(path, File::RDWR | File::CREAT) do |f|
-        f.flock(File::LOCK_EX)
-        begin
-          result = (yield get_data(f))
-          f.rewind
-          f.truncate(0)
-          if result
-            write(f, dump(result))
-          end
-        ensure
-          f.flock(File::LOCK_UN)
-        end
-      end
-    rescue Errno::ENOENT, Exception  => e
-      ScoutApm::Agent.instance.logger.error("Unable to access the layaway file [#{e.class} - #{e.message}]. " +
-                                            "The user running the app must have read & write access. " +
-                                            "Change the path by setting the `data_file` key in scout_apm.yml"
-                                           )
-      ScoutApm::Agent.instance.logger.debug(e.backtrace.join("\n\t"))
-
-      # ensure the in-memory metric hash is cleared so data doesn't continue to accumulate.
-      # ScoutApm::Agent.instance.store.metric_hash = {}
+    def write(data)
+      serialized_data = serialize(data)
+      File.open(path, "w") { |f| write_raw(f, serialized_data) }
     end
 
-    def get_data(f)
-      data = read_until_end(f)
-      result = load(data)
-      f.truncate(0)
-      result
+    def serialize(data)
+      Marshal.dump(data)
     end
 
-    def write(f, string)
-      result = 0
-      while (result < string.length)
-        result += f.write_nonblock(string)
-      end
-    rescue Errno::EAGAIN, Errno::EINTR
-      IO.select(nil, [f])
-      retry
+    def deserialize(data)
+      Marshal.load(data)
     end
 
-    def read_until_end(f)
+    def read_raw(f)
       contents = ""
       while true
         contents << f.read_nonblock(10_000)
@@ -99,5 +41,16 @@ module ScoutApm
     rescue EOFError
       contents
     end
+
+    def write_raw(f, data)
+      result = 0
+      while (result < data.length)
+        result += f.write_nonblock(data)
+      end
+    rescue Errno::EAGAIN, Errno::EINTR
+      IO.select(nil, [f])
+      retry
+    end
   end
 end
+

data/lib/scout_apm/store.rb

@@ -71,11 +71,10 @@ module ScoutApm
         reporting_periods.select { |time, rp| force || time.timestamp < current_timestamp.timestamp}.
                           each   { |time, rp|
                                    collect_samplers(rp)
-                                   layaway.add_reporting_period(time, rp)
+                                   layaway.write_reporting_period(rp)
                                    reporting_periods.delete(time)
                                  }
       }
-      ScoutApm::Agent.instance.logger.debug("Finished writing to layaway")
     end
 
     ######################################
@@ -108,8 +107,25 @@ module ScoutApm
       @timestamp = @raw_time.to_i - @raw_time.sec # The normalized time (integer) to compare by
     end
 
+    def self.minutes_ago(min, base_time=Time.now)
+      adjusted = base_time - (min * 60)
+      new(adjusted)
+    end
+
     def to_s
-      Time.at(@timestamp).iso8601
+      strftime
+    end
+
+    def strftime(pattern=nil)
+      if pattern.nil?
+        to_time.iso8601
+      else
+        to_time.strftime(pattern)
+      end
+    end
+
+    def to_time
+      Time.at(@timestamp)
     end
 
     def eql?(o)

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "1.6.1"
+  VERSION = "1.6.2"
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.6.2
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-16 00:00:00.000000000 Z
+date: 2016-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -77,10 +77,10 @@ files:
 - ".gitignore"
 - CHANGELOG.markdown
 - Gemfile
+- LICENSE.md
 - README.markdown
 - Rakefile
 - data/cacert.pem
-- lib/allocations.bundle
 - lib/scout_apm.rb
 - lib/scout_apm/agent.rb
 - lib/scout_apm/agent/logging.rb
@@ -181,7 +181,6 @@ files:
 - lib/scout_apm/utils/time.rb
 - lib/scout_apm/utils/unique_id.rb
 - lib/scout_apm/version.rb
-- lib/stacks.bundle
 - scout_apm.gemspec
 - test/data/config_test_1.yml
 - test/test_helper.rb