yabeda-delayed_job 0.1.0

data/example/grafana/grafana.ini

@@ -0,0 +1,9 @@
+[paths]
+provisioning = /etc/grafana/provisioning
+
+[server]
+http_port = 9091
+
+[users]
+default_theme = dark
+

data/example/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+

data/example/jobs.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class RandomJob
+  def call
+    sleep(rand(5000) / 1000.0)
+  end
+end
+
+class RandomFailJob
+  def call
+    raise StandardError if (rand(100) % 3).zero?
+  end
+end

data/example/log/delayed_job.log

@@ -0,0 +1,12 @@
+# Logfile created on 2021-06-21 18:52:52 +0000 by logger.rb/v1.4.3
+I, [2021-06-21T18:52:52.998760 #85]  INFO -- : 2021-06-21T18:52:52+0000: [Worker(delayed_job host:d7008330188a pid:85)] Starting job worker
+I, [2021-06-21T18:52:53.031345 #85]  INFO -- : 2021-06-21T18:52:53+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=1) RUNNING
+I, [2021-06-21T18:52:55.246040 #85]  INFO -- : 2021-06-21T18:52:55+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=1) COMPLETED after 2.2140
+I, [2021-06-21T18:52:55.249806 #85]  INFO -- : 2021-06-21T18:52:55+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=2) RUNNING
+I, [2021-06-21T18:52:59.055058 #85]  INFO -- : 2021-06-21T18:52:59+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=2) COMPLETED after 3.8045
+I, [2021-06-21T18:52:59.058246 #85]  INFO -- : 2021-06-21T18:52:59+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=3) RUNNING
+I, [2021-06-21T18:52:59.909979 #85]  INFO -- : 2021-06-21T18:52:59+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=3) COMPLETED after 0.8510
+I, [2021-06-21T18:52:59.914425 #85]  INFO -- : 2021-06-21T18:52:59+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=4) RUNNING
+I, [2021-06-21T18:53:00.916535 #85]  INFO -- : 2021-06-21T18:53:00+0000: [Worker(delayed_job host:d7008330188a pid:85)] Exiting...
+I, [2021-06-21T18:53:04.921236 #85]  INFO -- : 2021-06-21T18:53:04+0000: [Worker(delayed_job host:d7008330188a pid:85)] Job RandomJob#call (id=4) COMPLETED after 5.0059
+I, [2021-06-21T18:53:04.922786 #85]  INFO -- : 2021-06-21T18:53:04+0000: [Worker(delayed_job host:d7008330188a pid:85)] 4 jobs processed at 0.3355 j/s, 0 failed

data/example/prometheus/config.yml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval: 5s
+  scrape_timeout: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+- job_name: 'enqueuer'
+  scheme: http
+  metrics_path: '/metrics'
+  static_configs:
+    - targets:
+      - 'enqueuer:9394'
+- job_name: 'delayed_job'
+  scheme: http
+  metrics_path: '/metrics'
+  static_configs:
+    - targets:
+      - 'delayed_job:9394'
+

data/example/prometheus_store.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+FileUtils.rm_rf('/tmp/prometheus')
+
+require 'yabeda/prometheus'
+data_store = Prometheus::Client::DataStores::DirectFileStore.new(dir: '/tmp/prometheus')
+Prometheus::Client.config.data_store = data_store

data/lib/yabeda/delayed_job.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'yabeda/delayed_job/version'
+require 'delayed_job'
+require 'yabeda'
+require 'yabeda/delayed_job/plugin'
+
+module Yabeda
+  module DelayedJob
+    class Error < StandardError; end
+
+    LONG_RUNNING_JOB_RUNTIME_BUCKETS = [
+      0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, # standard (from Prometheus)
+      30, 60, 120, 300, 1800, 3600, 21_600 # DelayedJob tasks may be very long-running
+    ].freeze
+
+    ::Yabeda.configure do
+      group :delayed_job
+
+      counter :jobs_enqueued_total, tags: %i[queue worker], comment: 'A counter of the total number of jobs enqueued.'
+      counter :jobs_failed_total, tags: %i[queue worker error], comment: 'asdf'
+      counter :jobs_errored_total, tags: %i[queue worker error], comment: 'asdf'
+
+      histogram :job_runtime, comment: 'A histogram of the job execution time.',
+                              unit: :seconds, per: :job,
+                              tags: %i[queue worker],
+                              buckets: LONG_RUNNING_JOB_RUNTIME_BUCKETS
+      gauge :running_job_runtime, tags: %i[queue worker], aggregation: :max, unit: :seconds,
+                                  comment: 'How long currently running jobs are running ' \
+                                           '(useful for detection of hung jobs)'
+      gauge     :jobs_waiting_count,   tags: %i[queue], comment: 'The number of jobs waiting to process in sidekiq.'
+      gauge     :queue_latency,        tags: %i[queue],
+                                       comment: 'The queue latency, the difference in seconds since the oldest ' \
+                                                'job in the queue was enqueued'
+
+      collect do
+        if ::Yabeda::DelayedJob.server?
+          Yabeda::DelayedJob.track_max_job_runtime
+          ::Yabeda::DelayedJob.track_database_metrics if ::Yabeda::DelayedJob.active_record_adapter?
+        end
+      end
+    end
+
+    class << self
+      def server?
+        require 'delayed/command'
+        @server ||= ObjectSpace.each_object(Delayed::Command).any?
+      end
+
+      def labelize(job)
+        result = { queue: job.queue, worker: job.name }
+        result.merge!(error: job.error.class.name) if job.error
+        result
+      end
+
+      def track_database_metrics
+        job_scope.select(:queue).count.each do |queue, count|
+          Yabeda.delayed_job.jobs_waiting_count.set({ queue: queue }, count)
+        end
+        job_scope.select('queue, max(NOW() - run_at) latency').each do |job|
+          Yabeda.delayed_job.queue_latency.set({ queue: job.queue }, job.latency)
+        end
+      end
+
+      def job_scope
+        db_time_now = ::Delayed::Worker.backend.db_time_now
+        ::Delayed::Worker.backend.where(
+          '(run_at <= ? AND (locked_at IS NULL OR locked_at < ?)) AND failed_at IS NULL',
+          db_time_now,
+          db_time_now - ::Delayed::Worker.max_run_time
+        ).group(:queue)
+      end
+
+      def active_record_adapter?
+        defined?(Delayed::Backend::ActiveRecord::Job) &&
+          Delayed::Worker.backend.name == Delayed::Backend::ActiveRecord::Job.name
+      end
+
+      # Hash of hashes containing all currently running jobs' start timestamps
+      # to calculate maximum durations of currently running not yet completed jobs
+      # { { queue: "default", worker: "SomeJob" } => { "jid1" => 100500, "jid2" => 424242 } }
+      attr_accessor :jobs_started_at
+
+      def track_max_job_runtime
+        now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        ::Yabeda::DelayedJob.jobs_started_at.each do |labels, jobs|
+          oldest_job_started_at = jobs.values.min
+          oldest_job_duration = oldest_job_started_at ? (now - oldest_job_started_at).round(3) : 0
+          Yabeda.delayed_job.running_job_runtime.set(labels, oldest_job_duration)
+        end
+      end
+
+      def elapsed(start)
+        (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).round(3)
+      end
+    end
+
+    self.jobs_started_at = Concurrent::Hash.new { |hash, key| hash[key] = Concurrent::Hash.new }
+
+    Delayed::Worker.plugins << ::Yabeda::DelayedJob::Plugin
+  end
+end

data/lib/yabeda/delayed_job/plugin.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+
+module Yabeda
+  module DelayedJob
+    class Plugin < ::Delayed::Plugin
+      callbacks do |lifecycle|
+        lifecycle.after(:enqueue) do |job|
+          labels = ::Yabeda::DelayedJob.labelize(job)
+          ::Yabeda.delayed_job.jobs_enqueued_total.increment(labels)
+        end
+
+        lifecycle.after(:failure) do |_worker, job|
+          labels = ::Yabeda::DelayedJob.labelize(job)
+          ::Yabeda.delayed_job.jobs_failed_total.increment(labels)
+        end
+
+        lifecycle.after(:error) do |_worker, job|
+          labels = ::Yabeda::DelayedJob.labelize(job)
+          ::Yabeda.delayed_job.jobs_errored_total.increment(labels)
+        end
+
+        lifecycle.around(:perform) do |worker, job, &block|
+          labels = ::Yabeda::DelayedJob.labelize(job)
+
+          Process.clock_gettime(Process::CLOCK_MONOTONIC).tap do |start|
+            Yabeda::DelayedJob.jobs_started_at[labels][job.id] = start
+
+            block.call(worker)
+
+            Yabeda.delayed_job.job_runtime.measure(
+              labels,
+              ::Yabeda::DelayedJob.elapsed(start)
+            )
+          end
+        ensure
+          Yabeda::DelayedJob.jobs_started_at[labels].delete(job.id)
+        end
+      end
+    end
+  end
+end

data/lib/yabeda/delayed_job/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Yabeda
+  module DelayedJob
+    VERSION = '0.1.0'
+  end
+end

data/yabeda-delayed_job.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'yabeda/delayed_job/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'yabeda-delayed_job'
+  spec.version       = Yabeda::DelayedJob::VERSION
+  spec.authors       = ['Dmitry']
+  spec.email         = ['dsalahutdinov@gmail.com']
+
+  spec.summary       = 'Monitoring for delayed_job background worker'
+  spec.description   = 'Extends Yabeda to colleck delayed_job metrics'
+  spec.homepage      = 'https://github.com/yabeda-rb/yabeda-delayed_job'
+  spec.license       = 'MIT'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/yabeda-rb/yabeda-delayed_job'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'delayed_job'
+  spec.add_dependency 'yabeda'
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 1.15'
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: yabeda-delayed_job
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dmitry
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yabeda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+description: Extends Yabeda to colleck delayed_job metrics
+email:
+- dsalahutdinov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- docker-compose.yml
+- docs/yabeda-delayed_job-grafana.png
+- example/Gemfile
+- example/Gemfile.lock
+- example/delayed_job.rb
+- example/docker-compose.yml
+- example/enqueuer.rb
+- example/grafana/dashboards/delayed_job.json
+- example/grafana/grafana.ini
+- example/grafana/provisioning/dashboards/delayed_job.yml
+- example/grafana/provisioning/datasources/prometheus.yml
+- example/jobs.rb
+- example/log/delayed_job.log
+- example/prometheus/config.yml
+- example/prometheus_store.rb
+- lib/yabeda/delayed_job.rb
+- lib/yabeda/delayed_job/plugin.rb
+- lib/yabeda/delayed_job/version.rb
+- yabeda-delayed_job.gemspec
+homepage: https://github.com/yabeda-rb/yabeda-delayed_job
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/yabeda-rb/yabeda-delayed_job
+  source_code_uri: https://github.com/yabeda-rb/yabeda-delayed_job
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key: 
+specification_version: 4
+summary: Monitoring for delayed_job background worker
+test_files: []