sidekiq_queue_metrics 1.0 → 2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 616f7d90c5c36572b23161d8fcc09fdac5e05aa3fc4a4770445ecadd916854fa
-  data.tar.gz: e7ef6953e73d4c64a7bab64a3d6302be8f9aebd51152667be1815f76956d1a1c
+  metadata.gz: 8d104466f165a5787ee101d1a4d4b7e1909d47dafab3f7ea3363454f44a22f27
+  data.tar.gz: 6eeee50fdc2eedb29aa6adc216b1ede0249b57e49770638fb96300f0400c7b0f
 SHA512:
-  metadata.gz: 5457ea650cfb8a27175bd07a480695ac8105d4a31f4115a6fd2fd17b6b9e1fd4640deb332b94818eae24e24e487c4e7c4308839a3be776852d5e2bd6bda22284
-  data.tar.gz: 8a0425708609f2a8d0887b695261c2b9d354c038952cb597c26163c6f92e050aed980c0981c6667b4469c40dc3cc26c5518b3a02a229e7f1193bccc83f82f8b6
+  metadata.gz: bd32c75be538976be8659d35016c2d5eaabc36cabd33f2ddf351f4dd3289bd758d67e16f8b9fe912ca615c7167b9b8d683a4ae3278bbf6571f22d21f1fe0f673
+  data.tar.gz: 8916f90cc120083e3c66a0553b2b6eda9273bcc8a47148c000a7fd981f86f174e3d03094be9f0a8e2c9b0b410917bdf15bf0e3297722ecc561f56d79464492ba

data/README.md

@@ -37,8 +37,8 @@ Sidekiq::QueueMetrics.fetch
 Output:
 ```ruby
 {
-  "mailer_queue" => {"processed" => 5, "failed" => 1, "enqueued" => 2, "in_retry" => 0},
-  "default_queue" => {"processed" => 10, "failed" => 0, "enqueued" => 1, "in_retry" => 1}
+  "mailer_queue" => {"processed" => 5, "failed" => 1, "enqueued" => 2, "in_retry" => 0, "scheduled" => 0},
+  "default_queue" => {"processed" => 10, "failed" => 0, "enqueued" => 1, "in_retry" => 1, "scheduled" => 2}
 }
 ```
 

data/lib/sidekiq_queue_metrics/configuration.rb

@@ -11,6 +11,14 @@ module Sidekiq::QueueMetrics
     @storage_location = key
   end
 
+  def self.max_recently_failed_jobs=(count)
+    @max_recently_failed_jobs = count
+  end
+
+  def self.max_recently_failed_jobs
+    @max_recently_failed_jobs || 50
+  end
+
   def self.storage_location
     @storage_location
   end

data/lib/sidekiq_queue_metrics/monitor/job_death_monitor.rb

@@ -4,11 +4,15 @@ module Sidekiq::QueueMetrics
   class JobDeathMonitor < Monitor
     def self.proc
       Proc.new do |job, exception|
-        queue = job['queue']
-        JobDeathMonitor.new.monitor(queue)
+        JobDeathMonitor.new.monitor(job)
       end
     end
 
+    def monitor(job)
+      super(job['queue'])
+      Storage.add_failed_job(job)
+    end
+
     def status_counter
       'failed'
     end

data/lib/sidekiq_queue_metrics/queue_metrics.rb

@@ -46,6 +46,10 @@ module Sidekiq::QueueMetrics
       Sidekiq::ScheduledSet.new.group_by(&:queue).map {|queue, jobs| [queue, jobs.count]}.to_h
     end
 
+    def failed_jobs(queue)
+      Storage.failed_jobs(queue).reverse
+    end
+
     private
     def val_or_default(val, default = 0)
       val || default

data/lib/sidekiq_queue_metrics/storage.rb

@@ -1,5 +1,7 @@
 module Sidekiq::QueueMetrics
   class Storage
+    FAILED_JOBS_KEY = 'failed_jobs'.freeze
+
     class << self
       def set_stats(key = stats_key, value)
         Sidekiq.redis_pool.with do |conn|
@@ -13,6 +15,25 @@ module Sidekiq::QueueMetrics
         end
       end
 
+      def add_failed_job(job, max_count = Sidekiq::QueueMetrics.max_recently_failed_jobs)
+        Sidekiq.redis_pool.with do |conn|
+          queue = job['queue']
+          failed_jobs = JSON.parse(conn.get("#{FAILED_JOBS_KEY}:#{queue}") || '[]')
+
+          if failed_jobs.size >= max_count
+            (failed_jobs.size - max_count + 1).times {failed_jobs.shift}
+          end
+
+          conn.set("#{FAILED_JOBS_KEY}:#{queue}", (failed_jobs << job).to_json)
+        end
+      end
+
+      def failed_jobs(queue)
+        Sidekiq.redis_pool.with do |conn|
+          JSON.parse(conn.get("#{FAILED_JOBS_KEY}:#{queue}") || '[]')
+        end
+      end
+
       def stats_key
         Sidekiq::QueueMetrics.storage_location || 'queue_stats'
       end

data/lib/sidekiq_queue_metrics/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module QueueMetrics
-    VERSION = '1.0'
+    VERSION = '2.0'
   end
 end

data/lib/sidekiq_queue_metrics/views/failed_job.erb

@@ -0,0 +1,80 @@
+<header>
+  <h3><%= t('Job') %></h3>
+</header>
+
+<div class="table_container">
+  <table class="table table-bordered table-striped">
+    <tbody>
+    <tr>
+      <th><%= t('Queue') %></th>
+      <td>
+        <%= @job['queue'] %>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('Job') %></th>
+      <td>
+        <code><%= @job['class'] %></code>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('Arguments') %></th>
+      <td>
+        <code class="code-wrap">
+          <div class="args-extended"><%= @job['args'].join(', ') %></div>
+        </code>
+      </td>
+    </tr>
+    <tr>
+      <th>JID</th>
+      <td>
+        <code><%= @job['jid'] %></code>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('CreatedAt') %></th>
+      <td><%= relative_time(Time.at(@job['created_at'])) %></td>
+    </tr>
+    <tr>
+      <th><%= t('Enqueued') %></th>
+      <td><%= relative_time(Time.at(@job['enqueued_at'])) %></td>
+    </tr>
+    <% unless retry_extra_items(OpenStruct.new(item: @job)).empty? %>
+      <tr>
+        <th><%= t('Extras') %></th>
+        <td>
+          <code>
+            <%= retry_extra_items(OpenStruct.new(item: @job)).inspect %>
+          </code>
+        </td>
+      </tr>
+    <% end %>
+    </tbody>
+  </table>
+
+  <h3><%= t('Error') %></h3>
+  <div class="table_container">
+    <table class="error table table-bordered table-striped">
+      <tbody>
+      <tr>
+        <th><%= t('ErrorClass') %></th>
+        <td>
+          <code><%= @job['error_class'] %></code>
+        </td>
+      </tr>
+      <tr>
+        <th><%= t('ErrorMessage') %></th>
+        <td><%= h(@job['error_message']) %></td>
+      </tr>
+      <% if !@job['error_backtrace'].nil? %>
+        <tr>
+          <th><%= t('ErrorBacktrace') %></th>
+          <td>
+            <code><%= @job['error_backtrace'].join("<br/>") %></code>
+          </td>
+        </tr>
+      <% end %>
+      </tbody>
+    </table>
+  </div>
+</div>

data/lib/sidekiq_queue_metrics/views/queue_summary.erb

@@ -0,0 +1,70 @@
+<style>
+  .queue_summary_bar {
+    padding-left: 0;
+    padding-right: 0;
+  }
+
+  .queue_summary_bar ul li {
+    width: 19%;
+  }
+</style>
+
+<header class="row">
+  <div class="col-sm-10">
+    <h3><%= @queue %></h3>
+  </div>
+</header>
+
+<div class="col-sm-12 summary_bar queue_summary_bar">
+  <ul class="list-unstyled summary row">
+    <li class="processed col-sm-1">
+      <span class="count"><%= number_with_delimiter(@queue_stats['processed']) %></span>
+      <span class="desc"><%= t('Processed') %></span>
+    </li>
+    <li class="failed col-sm-1">
+      <span class="count"><%= number_with_delimiter(@queue_stats['failed']) %></span>
+      <span class="desc"><%= t('Failed') %></span>
+    </li>
+    <li class="enqueued col-sm-1">
+      <span class="count"><%= number_with_delimiter(@queue_stats['enqueued']) %></span>
+      <span class="desc"><%= t('Enqueued') %></span>
+    </li>
+    <li class="retries col-sm-1">
+      <span class="count"><%= number_with_delimiter(@queue_stats['in_retry']) %></span>
+      <span class="desc"><%= t('Retries') %></span>
+    </li>
+    <li class="scheduled col-sm-1">
+      <span class="count"><%= number_with_delimiter(@queue_stats['scheduled']) %></span>
+      <span class="desc"><%= t('Scheduled') %></span>
+    </li>
+  </ul>
+</div>
+
+<header class="row">
+  <div class="col-sm-10">
+    <h3>Failed Jobs</h3>
+  </div>
+</header>
+
+<table class="table table-striped table-bordered table-white">
+  <thead>
+  <tr>
+    <th><%= t('Job') %></th>
+    <th><%= t('Arguments') %></th>
+    <th><%= t('Enqueued') %></th>
+    <th><%= t('Error') %></th>
+  </tr>
+  </thead>
+  <% @failed_jobs.each do |job| %>
+    <tr>
+      <td><%= job['class'] %></td>
+      <td><%= job['args'].join(', ') %></td>
+      <td>
+        <a href="<%= "/sidekiq/queue_metrics/queues/#{@queue}/jobs/#{job['jid']}" %>">
+          <%= relative_time(Time.at(job['enqueued_at'])) %>
+        </a>
+      </td>
+      <td><%= job['error_message'] %></td>
+    </tr>
+  <% end %>
+</table>

data/lib/sidekiq_queue_metrics/views/{queue_metrics.erb → queues_stats.erb}

@@ -38,8 +38,12 @@
   <table class="table table-striped table-bordered table-white queue_metrics">
     <thead>
     <tr>
-      <th><span class="heading"><%= queue %></span></th>
-      <th style="width: 30%"  class="center"><span class="heading">Count</span></th>
+      <th>
+        <a href="<%= "/sidekiq/queue_metrics/queues/#{queue}/summary" %>">
+          <span class="heading"><%= queue %></span>
+        </a>
+      </th>
+      <th style="width: 30%" class="center"><span class="heading">Count</span></th>
     </tr>
     </thead>
     <% metrics.each do |metric_name, count| %>

data/lib/sidekiq_queue_metrics/web_extension.rb

@@ -5,7 +5,22 @@ module Sidekiq::QueueMetrics
 
       app.get "/queue_metrics" do
         @queue_metrics = Sidekiq::QueueMetrics.fetch
-        render(:erb, File.read(File.join(view_path, "queue_metrics.erb")))
+        render(:erb, File.read(File.join(view_path, "queues_stats.erb")))
+      end
+
+      app.get '/queue_metrics/queues/:queue/summary' do
+        @queue = route_params[:queue]
+        @queue_stats = Sidekiq::QueueMetrics.fetch[@queue]
+        @failed_jobs = Sidekiq::QueueMetrics.failed_jobs(@queue)
+        render(:erb, File.read(File.join(view_path, "queue_summary.erb")))
+      end
+
+      app.get '/queue_metrics/queues/:queue/jobs/:jid' do
+        queue = route_params[:queue]
+        jid = route_params[:jid]
+        failed_jobs = Sidekiq::QueueMetrics.failed_jobs(queue)
+        @job = failed_jobs.find {|job| job['jid'] == jid}
+        render(:erb, File.read(File.join(view_path, "failed_job.erb")))
       end
     end
   end

data/spec/lib/sidekiq_queue_metrics/monitor/job_death_monitor_spec.rb

@@ -8,6 +8,7 @@ describe Sidekiq::QueueMetrics::JobDeathMonitor do
       it 'should create stats key and add stats of queue' do
         expect(Sidekiq::QueueMetrics::Storage).to receive(:get_stats).and_return(nil)
         expect(Sidekiq::QueueMetrics::Storage).to receive(:set_stats).with({mailer_queue: {failed: 1}}.to_json)
+        expect(Sidekiq::QueueMetrics::Storage).to receive(:add_failed_job).with(job)
 
         monitor.call(job)
       end
@@ -15,13 +16,15 @@ describe Sidekiq::QueueMetrics::JobDeathMonitor do
 
     context 'when stats exists' do
       it 'should create a new queue when it does not exist' do
+        job_queue = {'queue' => 'job_queue'}
         existing_stats = {mailer_queue: {failed: 1}}.to_json
         expected_stats = {mailer_queue: {failed: 1}, job_queue: {failed: 1}}.to_json
 
         expect(Sidekiq::QueueMetrics::Storage).to receive(:get_stats).and_return(existing_stats)
         expect(Sidekiq::QueueMetrics::Storage).to receive(:set_stats).with(expected_stats)
+        expect(Sidekiq::QueueMetrics::Storage).to receive(:add_failed_job).with(job_queue)
 
-        monitor.call({'queue' => 'job_queue'})
+        monitor.call(job_queue)
       end
 
       it 'should update existing queue' do
@@ -30,6 +33,7 @@ describe Sidekiq::QueueMetrics::JobDeathMonitor do
 
         expect(Sidekiq::QueueMetrics::Storage).to receive(:get_stats).and_return(existing_stats)
         expect(Sidekiq::QueueMetrics::Storage).to receive(:set_stats).with(expected_stats)
+        expect(Sidekiq::QueueMetrics::Storage).to receive(:add_failed_job).with(job)
 
         monitor.call(job)
       end
@@ -40,6 +44,7 @@ describe Sidekiq::QueueMetrics::JobDeathMonitor do
 
         expect(Sidekiq::QueueMetrics::Storage).to receive(:get_stats).and_return(existing_stats)
         expect(Sidekiq::QueueMetrics::Storage).to receive(:set_stats).with(expected_stats)
+        expect(Sidekiq::QueueMetrics::Storage).to receive(:add_failed_job).with(job)
 
         monitor.call(job)
       end

data/spec/lib/sidekiq_queue_metrics/{queue_stats_spec.rb → queue_metrics_spec.rb}

@@ -51,10 +51,11 @@ describe Sidekiq::QueueMetrics do
     end
 
     it 'should return Sidekiq::QueueMetrics for all sidekiq queues' do
-      jobs_in_retry_queue = []
+      jobs_in_retry_queue = scheduled_jobs = []
 
       expect(Sidekiq::QueueMetrics::Storage).to receive(:get_stats).and_return(nil)
       expect(Sidekiq::RetrySet).to receive(:new).and_return(jobs_in_retry_queue)
+      expect(Sidekiq::ScheduledSet).to receive(:new).and_return(scheduled_jobs)
       expect(Sidekiq::Queue).to receive(:new).with('mailer_queue').and_return(OpenStruct.new(size: 0))
       expect(Sidekiq::Queue).to receive(:new).with('heavy_jobs_queue').and_return(OpenStruct.new(size: 0))
 
@@ -73,4 +74,15 @@ describe Sidekiq::QueueMetrics do
       expect(queue_stats['heavy_jobs_queue']['scheduled']).to be_zero
     end
   end
+
+  describe '#failed_jobs' do
+    it 'should return failed jobs for a queue' do
+      queue = 'default_queue'
+      job_1 = double(:job)
+      job_2 = double(:job)
+      expect(Sidekiq::QueueMetrics::Storage).to receive(:failed_jobs).and_return([job_1, job_2])
+
+      expect(Sidekiq::QueueMetrics.failed_jobs(queue)).to eq([job_2, job_1])
+    end
+  end
 end

data/spec/lib/sidekiq_queue_metrics/storage_spec.rb

@@ -0,0 +1,82 @@
+describe Sidekiq::QueueMetrics::Storage do
+  class MockRedisPool
+    attr_reader :conn
+
+    def initialize(conn)
+      @conn = conn
+    end
+
+    def with
+      yield conn
+    end
+  end
+
+  let(:mock_redis_conn) {double(:connection)}
+  let(:mock_redis_pool) {MockRedisPool.new(mock_redis_conn)}
+
+  describe '#add_failed_job' do
+    it 'should add first failed job' do
+      job = {'queue' => 'mailer_queue'}
+      expect(Sidekiq).to receive(:redis_pool).and_return(mock_redis_pool)
+      expect(mock_redis_conn).to receive(:get).with("failed_jobs:mailer_queue").and_return(nil)
+
+      expect(mock_redis_conn).to receive(:set).with("failed_jobs:mailer_queue", [job].to_json)
+
+      Sidekiq::QueueMetrics::Storage.add_failed_job(job)
+    end
+
+    it 'should add failed job to existing jobs' do
+      key = "failed_jobs:mailer_queue"
+      new_job = {'queue' => 'mailer_queue', 'args' => [1]}
+      existing_jobs = [{'queue' => 'mailer_queue', 'args' => [2]}]
+
+      expect(Sidekiq).to receive(:redis_pool).and_return(mock_redis_pool)
+      expect(mock_redis_conn).to receive(:get).with(key).and_return(existing_jobs.to_json)
+
+      expect(mock_redis_conn).to receive(:set).with(key, [existing_jobs.first, new_job].to_json)
+
+      Sidekiq::QueueMetrics::Storage.add_failed_job(new_job)
+    end
+
+    it 'should delete old job when failed jobs limit has reached' do
+      key = "failed_jobs:mailer_queue"
+      new_job = {'queue' => 'mailer_queue', 'args' => [1]}
+      oldest_job = {'queue' => 'mailer_queue', 'args' => [2]}
+      older_job = {'queue' => 'mailer_queue', 'args' => [3]}
+
+      existing_jobs = [oldest_job, older_job]
+
+      expect(Sidekiq).to receive(:redis_pool).and_return(mock_redis_pool)
+      expect(mock_redis_conn).to receive(:get).with(key).and_return(existing_jobs.to_json)
+
+      expect(mock_redis_conn).to receive(:set).with(key, [older_job, new_job].to_json)
+
+      Sidekiq::QueueMetrics::Storage.add_failed_job(new_job, 2)
+    end
+  end
+
+  describe '#failed_jobs' do
+    context 'when failed jobs are not present' do
+      it 'should return failed jobs for a given queue' do
+        queue = 'mailer_queue'
+        expect(Sidekiq).to receive(:redis_pool).and_return(mock_redis_pool)
+
+        expect(mock_redis_conn).to receive(:get).with("failed_jobs:#{queue}").and_return(nil)
+
+        expect(Sidekiq::QueueMetrics::Storage.failed_jobs(queue)).to be_empty
+      end
+    end
+
+    context 'when failed jobs are present' do
+      it 'should return failed jobs for a given queue' do
+        queue = 'mailer_queue'
+        jobs = [{'queue' => 'mailer_queue', 'args' => [1]}]
+        expect(Sidekiq).to receive(:redis_pool).and_return(mock_redis_pool)
+
+        expect(mock_redis_conn).to receive(:get).with("failed_jobs:#{queue}").and_return(jobs.to_json)
+
+        expect(Sidekiq::QueueMetrics::Storage.failed_jobs(queue)).to eq(jobs)
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq_queue_metrics
 version: !ruby/object:Gem::Version
-  version: '1.0'
+  version: '2.0'
 platform: ruby
 authors:
 - Ajit Singh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-14 00:00:00.000000000 Z
+date: 2018-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -87,12 +87,15 @@ files:
 - lib/sidekiq_queue_metrics/queue_metrics.rb
 - lib/sidekiq_queue_metrics/storage.rb
 - lib/sidekiq_queue_metrics/version.rb
-- lib/sidekiq_queue_metrics/views/queue_metrics.erb
+- lib/sidekiq_queue_metrics/views/failed_job.erb
+- lib/sidekiq_queue_metrics/views/queue_summary.erb
+- lib/sidekiq_queue_metrics/views/queues_stats.erb
 - lib/sidekiq_queue_metrics/web_extension.rb
 - sidekiq_queue_metrics.gemspec
 - spec/lib/sidekiq_queue_metrics/monitor/job_death_monitor_spec.rb
 - spec/lib/sidekiq_queue_metrics/monitor/job_success_monitor_spec.rb
-- spec/lib/sidekiq_queue_metrics/queue_stats_spec.rb
+- spec/lib/sidekiq_queue_metrics/queue_metrics_spec.rb
+- spec/lib/sidekiq_queue_metrics/storage_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/ajitsing/sidekiq_queue_metrics
 licenses:
@@ -121,5 +124,6 @@ summary: Records stats of each sidekiq queue and exposes APIs to retrieve them
 test_files:
 - spec/lib/sidekiq_queue_metrics/monitor/job_death_monitor_spec.rb
 - spec/lib/sidekiq_queue_metrics/monitor/job_success_monitor_spec.rb
-- spec/lib/sidekiq_queue_metrics/queue_stats_spec.rb
+- spec/lib/sidekiq_queue_metrics/queue_metrics_spec.rb
+- spec/lib/sidekiq_queue_metrics/storage_spec.rb
 - spec/spec_helper.rb