inst-jobs-statsd 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ea3cc2f583c0d355a0ab7440affd123343fa8de1
+  data.tar.gz: 7fca023059959468544941e582d9d805caf2f3e1
+SHA512:
+  metadata.gz: 116fada163459dab67b7d70a00d0a8191438636bffe7fa5c95e25b06ad45a1ac8a60e28f628b50333821f109cdca3e182d75a1eb145af54b9992b48d1f067a46
+  data.tar.gz: fe822cd09c1421b0d694b2ea8255e34f47e98b402275e9ba4c4b262f1b953127a00f122b58376299971c02a2b1fec9305529927db5ad49a817df55ca70749ef7

data/lib/inst-jobs-statsd.rb

@@ -0,0 +1,26 @@
+require 'inst-jobs'
+require 'inst_statsd'
+
+require_relative 'inst_jobs_statsd/version'
+
+require_relative 'inst_jobs_statsd/default_tracking'
+require_relative 'inst_jobs_statsd/jobs_tracker'
+
+require_relative 'inst_jobs_statsd/naming'
+
+require_relative 'inst_jobs_statsd/stats/counters'
+require_relative 'inst_jobs_statsd/stats/counters/failed'
+require_relative 'inst_jobs_statsd/stats/counters/orphaned'
+require_relative 'inst_jobs_statsd/stats/counters/run'
+
+require_relative 'inst_jobs_statsd/stats/periodic'
+require_relative 'inst_jobs_statsd/stats/periodic/failed'
+require_relative 'inst_jobs_statsd/stats/periodic/queue'
+require_relative 'inst_jobs_statsd/stats/periodic/run'
+
+require_relative 'inst_jobs_statsd/stats/timing'
+require_relative 'inst_jobs_statsd/stats/timing/failed'
+require_relative 'inst_jobs_statsd/stats/timing/perform'
+require_relative 'inst_jobs_statsd/stats/timing/pop'
+
+::InstStatsd::DefaultTracking.include InstJobsStatsd::DefaultTracking

data/lib/inst_jobs_statsd/default_tracking.rb

@@ -0,0 +1,15 @@
+# Defines InstStatsd::DefaultTracking.track_jobs
+# to be consistent with InstStatsd::DefaultTracking.track_sql etc
+module InstJobsStatsd
+  module DefaultTracking
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def track_jobs
+        @jobs_tracker ||= JobsTracker.new
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/jobs_tracker.rb

@@ -0,0 +1,25 @@
+module InstJobsStatsd
+  class JobsTracker
+    def self.track
+      @current_tracking = new
+      yield
+      tracking = @current_tracking
+      @current_tracking = nil
+      tracking
+    end
+
+    def initialize
+      Stats::Counters::Failed.enable
+      Stats::Counters::Orphaned.enable
+      Stats::Counters::Run.enable
+
+      Stats::Periodic::Failed.enable
+      Stats::Periodic::Queue.enable
+      Stats::Periodic::Run.enable
+
+      Stats::Timing::Failed.enable
+      Stats::Timing::Perform.enable
+      Stats::Timing::Pop.enable
+    end
+  end
+end

data/lib/inst_jobs_statsd/naming.rb

@@ -0,0 +1,49 @@
+module InstJobsStatsd
+  module Naming
+    BASENAME = 'delayedjob'.freeze
+
+    # The root prefix for all stat names
+    # TODO: Make this configurable
+    def self.basename
+      BASENAME
+    end
+
+    def self.qualified_names(stat_name, job)
+      names = ["#{basename}.#{stat_name}"]
+      tagged = tagged_stat(names[0], job)
+      names << tagged if tagged.present?
+      names
+    end
+
+    # Given a stat name, add a suffix to it to make it
+    # unique per job type -- using the job's class name
+    # and method name as appropriate
+    def self.tagged_stat(stat_name, job)
+      return unless job
+
+      obj_tag, method_tag = job_tags(job)
+      return if obj_tag.blank?
+
+      tagged = "#{stat_name}.tag.#{obj_tag}"
+      tagged += ".#{method_tag}" if method_tag.present?
+      tagged
+    end
+
+    # this converts Foo#bar" or "Foo.bar" into "Foo and "bar",
+    # and makes sure the values are valid to be used for statsd names
+    def self.job_tags(job)
+      return unless job
+      return unless job.tag
+      return if job.tag =~ /Class:0x/
+
+      obj_tag, method_tag = job.tag.split(/[\.#]/, 2).map do |v|
+        InstStatsd::Statsd.escape(v).gsub('::', '-')
+      end
+
+      tags = [obj_tag]
+      tags << method_tag if method_tag.present?
+
+      tags
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/counters/failed.rb

@@ -0,0 +1,28 @@
+module InstJobsStatsd
+  module Stats
+    module Counters
+      module Failed
+        def self.enable
+          enable_failed_count
+        end
+
+        def self.enable_failed_count
+          return if Delayed::Job::Failed < AfterCreateHook
+          Delayed::Job::Failed.include AfterCreateHook
+        end
+
+        module AfterCreateHook
+          def self.included(base)
+            base.after_create do
+              InstJobsStatsd::Stats::Counters::Failed.report_failed_count(self)
+            end
+          end
+        end
+
+        def self.report_failed_count(job)
+          Counters.report_count(:failed, 1, job: job)
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/counters/orphaned.rb

@@ -0,0 +1,34 @@
+module InstJobsStatsd
+  module Stats
+    module Counters
+      module Orphaned
+        def self.enable
+          enable_orphaned_count
+        end
+
+        # The idea of the orphaned count: when a job finishes, if there
+        # are other jobs locked_by the *same* worker, they must have been
+        # orphaned, because they are not going to be picked up and run by
+        # the worker -- the work queue is designed to only have one job
+        # locked_by a worker at a time.
+        # This is based on the symptom seen in AMS-447, where mutliple
+        # rows of the jobs table can be (incorrectly) updated by the same
+        # query.
+        def self.enable_orphaned_count
+          Delayed::Worker.lifecycle.before(:perform) do |_worker, job|
+            report_orphaned_count(job)
+          end
+        end
+
+        def self.report_orphaned_count(job)
+          scope = Delayed::Job.where(
+            'locked_by = ? AND locked_at = ? AND id <> ?',
+            job.locked_by, job.locked_at, job.id
+          )
+          count = scope.count
+          Counters.report_count(:orphaned, count, job: job) unless count.zero?
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/counters/run.rb

@@ -0,0 +1,21 @@
+module InstJobsStatsd
+  module Stats
+    module Counters
+      module Run
+        def self.enable
+          enable_run_count
+        end
+
+        def self.enable_run_count
+          Delayed::Worker.lifecycle.before(:perform) do |_worker, job|
+            report_run_count(job)
+          end
+        end
+
+        def self.report_run_count(job)
+          Counters.report_count(:run, 1, job: job)
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/counters.rb

@@ -0,0 +1,10 @@
+module InstJobsStatsd
+  module Stats
+    module Counters
+      def self.report_count(stat, value, job: nil, sample_rate: 1)
+        stats = Naming.qualified_names(stat, job)
+        InstStatsd::Statsd.count(stats, value, sample_rate)
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/periodic/failed.rb

@@ -0,0 +1,21 @@
+module InstJobsStatsd
+  module Stats
+    module Periodic
+      module Failed
+        def self.enable
+          enable_failed_depth
+        end
+
+        def self.enable_failed_depth
+          Periodic.enable_callbacks
+          Periodic.add(-> { report_failed_depth })
+        end
+
+        def self.report_failed_depth
+          count = Delayed::Job::Failed.count
+          Periodic.report_gauge(:failed_depth, count)
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/periodic/queue.rb

@@ -0,0 +1,49 @@
+module InstJobsStatsd
+  module Stats
+    module Periodic
+      module Queue
+        def self.enable
+          enable_queue_depth
+          enable_queue_age
+        end
+
+        def self.enable_queue_depth
+          Periodic.enable_callbacks
+          Periodic.add(-> { report_queue_depth })
+        end
+
+        def self.enable_queue_age
+          Periodic.enable_callbacks
+          Periodic.add(-> { report_queue_age })
+        end
+
+        def self.report_queue_depth
+          # count = Delayed::Job.jobs_count(:current)  <-- includes running / locked
+          scope = queued_jobs_scope
+          count = scope.count
+          Periodic.report_gauge(:queue_depth, count)
+        end
+
+        # Limit the jobs included in this gauge to prevent blowing up
+        # memory usage in iterating the list.
+        # This has the adverse effect of artificially capping this
+        # metric, but the limit should be high enough so that the
+        # the metric still has a meaningful range -- and even if
+        # the count is capped, the metric will continue to grow
+        # if the queue is actually stalled
+        def self.report_queue_age
+          jobs_run_at = queued_jobs_scope.limit(10_000).pluck(:run_at)
+          age_secs = jobs_run_at.map { |t| Delayed::Job.db_time_now - t }
+          Periodic.report_gauge(:queue_age_total, age_secs.sum)
+          Periodic.report_gauge(:queue_age_max, age_secs.max)
+        end
+
+        def self.queued_jobs_scope
+          Delayed::Job
+            .current
+            .where("locked_at IS NULL OR locked_by = 'on_hold'") # not running
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/periodic/run.rb

@@ -0,0 +1,38 @@
+module InstJobsStatsd
+  module Stats
+    module Periodic
+      module Run
+        def self.enable
+          enable_run_depth
+          enable_run_age
+        end
+
+        def self.enable_run_depth
+          Periodic.enable_callbacks
+          Periodic.add(-> { report_run_depth })
+        end
+
+        def self.enable_run_age
+          Periodic.enable_callbacks
+          Periodic.add(-> { report_run_age })
+        end
+
+        def self.report_run_depth
+          scope = running_jobs_scope
+          Periodic.report_gauge(:run_depth, scope.count)
+        end
+
+        def self.report_run_age
+          jobs_run_at = running_jobs_scope.limit(10_000).pluck(:run_at)
+          age_secs = jobs_run_at.map { |t| Delayed::Job.db_time_now - t }
+          Periodic.report_gauge(:run_age_total, age_secs.sum)
+          Periodic.report_gauge(:run_age_max, age_secs.max)
+        end
+
+        def self.running_jobs_scope
+          Delayed::Job.running
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/periodic.rb

@@ -0,0 +1,91 @@
+module InstJobsStatsd
+  module Stats
+    module Periodic
+      def self.enable_callbacks
+        @instance ||= Callbacks.new
+      end
+
+      def self.add(proc)
+        return unless @instance
+        @instance.add(proc)
+      end
+
+      def self.report_gauge(stat, value, job: nil, sample_rate: 1)
+        stats = Naming.qualified_names(stat, job)
+        InstStatsd::Statsd.gauge(stats, value, sample_rate)
+      end
+
+      class Callbacks
+        def initialize(min_interval = 60)
+          @timer = Timer.new(min_interval)
+          @procs = []
+          register_lifecycle
+        end
+
+        def add(proc)
+          @procs << proc if proc
+        end
+
+        # This hooks into the lifecycle events such that it will
+        # get triggered periodically which reasonable certainty
+        # -- as long as the rest of the inst-jobs processing
+        # system is working, any way.
+        # This allows for stats to be reported periodically
+        # without having to start a separate sideband process.
+        # It means that reporting of those stats will run
+        # inline (in the same process and thread) as the
+        # regular job processing work, but it also means
+        # no need for an additional database connection, and
+        # no managing of additional threads or processes.
+        #
+        # The :work_queue_pop event is used becaused in production
+        # mode, the 'parent_process' work queue is typically
+        # going to be used, and this callback runs in the parent
+        # process -- as opposed to having this callback run in
+        # each of the subordinate worker processes, which would
+        # not be ideal. In a dev env with the 'in_process' work queue,
+        # there's typically only going to be a single worker process
+        # anyway, so it works just as well.
+        def register_lifecycle
+          Delayed::Worker.lifecycle.after(:work_queue_pop) do |_q, _c|
+            @timer.tick do
+              run
+            end
+          end
+        end
+
+        def run
+          InstStatsd::Statsd.batch do
+            @procs.each(&:call)
+          end
+        end
+      end
+
+      class Timer
+        def initialize(min_interval)
+          @min_interval = min_interval * 1.0
+          @start_time = Delayed::Job.db_time_now
+          update_next_run
+        end
+
+        # This is called as often as possible, based on the lifecycle callbacks.
+        # When the required interval of time has passed, execute the given block
+        def tick
+          return unless Delayed::Job.db_time_now >= @next_run
+          update_next_run
+          yield
+        end
+
+        private
+
+        # Target the next run time to based on the original start time,
+        # instead of just adding the run interval, to prevent drift
+        # from the target interval as much as possible
+        def update_next_run
+          ticks = ((Delayed::Job.db_time_now - @start_time) / @min_interval).floor
+          @next_run = @start_time + (ticks + 1) * @min_interval
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/timing/failed.rb

@@ -0,0 +1,17 @@
+module InstJobsStatsd
+  module Stats
+    module Timing
+      module Failed
+        def self.enable
+          enable_failure_timing
+        end
+
+        def self.enable_failure_timing
+          Delayed::Worker.lifecycle.before(:error) do |_worker, job, _exception|
+            Timing.report_job_timing_failed(job)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/timing/perform.rb

@@ -0,0 +1,29 @@
+module InstJobsStatsd
+  module Stats
+    module Timing
+      module Perform
+        def self.enable
+          enable_batching
+          enable_perform_timing
+        end
+
+        def self.enable_batching
+          Delayed::Worker.lifecycle.around(:perform) do |worker, job, &block|
+            InstStatsd::Statsd.batch do
+              block.call(worker, job)
+            end
+          end
+        end
+
+        def self.enable_perform_timing
+          Delayed::Worker.lifecycle.around(:perform) do |worker, job, &block|
+            Timing.report_job_timing_queued(job)
+            Timing.report_timing(:perform, job: job) do
+              block.call(worker, job)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/timing/pop.rb

@@ -0,0 +1,28 @@
+module InstJobsStatsd
+  module Stats
+    module Timing
+      module Pop
+        def self.enable
+          enable_pop_timing
+          enable_workqueue_pop_timing
+        end
+
+        def self.enable_pop_timing
+          Delayed::Worker.lifecycle.around(:pop) do |worker, &block|
+            Timing.report_timing(:pop) do
+              block.call(worker)
+            end
+          end
+        end
+
+        def self.enable_workqueue_pop_timing
+          Delayed::Worker.lifecycle.around(:work_queue_pop) do |worker, config, &block|
+            Timing.report_timing(:workqueuepop) do
+              block.call(worker, config)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/stats/timing.rb

@@ -0,0 +1,27 @@
+module InstJobsStatsd
+  module Stats
+    module Timing
+      def self.report_timing(stat, job: nil, timing: nil, sample_rate: 1)
+        stats = Naming.qualified_names(stat, job)
+
+        if block_given?
+          InstStatsd::Statsd.time(stats, sample_rate) { yield }
+        else
+          InstStatsd::Statsd.timing(stats, timing, sample_rate)
+        end
+      end
+
+      def self.report_job_timing_queued(job)
+        return unless job
+        time_in_queue = ((Delayed::Job.db_time_now - job.run_at) * 1000).round
+        report_timing(:queue, job: job, timing: time_in_queue)
+      end
+
+      def self.report_job_timing_failed(job)
+        return unless job
+        time_to_failure = ((Delayed::Job.db_time_now - job.run_at) * 1000).round
+        report_timing(:failed_after, job: job, timing: time_to_failure)
+      end
+    end
+  end
+end

data/lib/inst_jobs_statsd/version.rb

@@ -0,0 +1,3 @@
+module InstJobsStatsd
+  VERSION = '1.0.0'.freeze
+end

data/spec/factories/jobs.rb

@@ -0,0 +1,9 @@
+FactoryGirl.define do
+  class JobFixture
+    attr_accessor :tag, :run_at
+  end
+
+  factory :job_fixture, aliases: [:job] do
+    tag 'Test::Job.perform'
+  end
+end

data/spec/factories/workers.rb

@@ -0,0 +1,9 @@
+FactoryGirl.define do
+  class WorkerFixture
+    attr_accessor :name
+  end
+
+  factory :worker_fixture, aliases: [:worker] do
+    sequence(:name) { |n| "worker-#{n}" }
+  end
+end

data/spec/gemfiles/42.gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+gemspec :path=>"../../"
+
+gem "rails", "~> 4.2.5"
+gem "after_transaction_commit", "<2"
+gem 'test_after_commit', '0.4.1'