kthxbye 1.2.1 → 1.3.0

data/lib/kthxbye.rb

@@ -11,6 +11,7 @@ require 'kthxbye/helper'
 require 'kthxbye/job'
 require 'kthxbye/worker'
 require 'kthxbye/failure'
+require 'kthxbye/stats'
 
 require 'kthxbye/version'
 
@@ -213,7 +214,8 @@ module Kthxbye
       :workers => workers.size,
       :working => working.size,
       :queues => queues.size,
-      :failed => Failure.count,
+      :failed => Stats["failures"],
+      :jobs_processed => Stats["processed"],
       :pending => queues.inject(0) {|m,o| m + size(o)}
     }
   end

data/lib/kthxbye/failure.rb

@@ -58,5 +58,9 @@ module Kthxbye
       redis.hdel( :failed, id ) 
     end
 
+    def self.clear_all
+      redis.del( :failed )
+    end
+
   end
 end

data/lib/kthxbye/job.rb

@@ -57,7 +57,7 @@ module Kthxbye
       # remove the job's data as well
       redis.hdel("data-store:#{queue}", id) 
       redis.hdel("result-store:#{queue}", id)
-      redis.hdel( :faulure, id )
+      redis.hdel( :failure, id )
       
       return ret
     end
@@ -69,7 +69,7 @@ module Kthxbye
       @id = id.to_i
       @queue = queue
       @data = data
-      @failed_attempts =  Failure.fails_for_job(@id) # local tracking only, for rerun purposes
+      @failed_attempts = Failure.fails_for_job(@id) # local tracking only, for rerun purposes
     end
 
     # Simply requeues the job to be rerun.
@@ -124,14 +124,19 @@ module Kthxbye
         redis.hset( "result-store:#{@queue}", @id, encode( result ) )
         return result
       rescue Object => ex
-        @failed_attempts += 1
-        log "Error occured: #{ex.message}. Try: #{@failed_attempts}/#{Kthxbye::Config.options[:attempts]}"
-        redis.publish("job.failed", @id)
-        return Kthxbye::Failure.create( self, ex ) if @failed_attempts >= Kthxbye::Config.options[:attempts]
-        perform
+        # handled by worker
+        raise ex
       end
     end
 
+    def fail(ex)
+      @failed_attempts += 1
+      log "Error occured: #{ex.message}. Try: #{@failed_attempts}/#{Kthxbye::Config.options[:attempts]}"
+      redis.publish("job.failed", @id)
+      Stats.incr("failures")
+      Kthxbye::Failure.create( self, ex ) 
+    end
+
     # Removes the job from the inactive queue.
     def active
       redis.srem("jobs:inactive", @id)

data/lib/kthxbye/stats.rb

@@ -0,0 +1,39 @@
+# Simple library to aid in tracking statistics regarding Kthxbye and its workers.
+# Submits jobs processed stats, total failures, total uptime, etc.
+# Puts all stats in stats:x vars in Redis. Super flexible, made to allow tracking
+# of any kind of count internally
+#
+# Built in stats (tracked by Kthxbye internals) include the following
+#
+#  processed - the count of processed jobs
+#  <worker>:processed - the count a given worker has processed
+#  failures - the count of failed jobs
+
+module Kthxbye
+  module Stats
+    extend self
+    extend Helper
+
+    # get count for a given stat (can also use [] method)
+    def get( stat )
+      redis.get( "stat:#{stat}" ).to_i
+    end
+    alias_method :[], :get
+
+    # increment a count by a given value (defaults to 1)
+    def incr( stat, by=1 )
+      redis.incrby("stat:#{stat}", by)
+    end
+
+    # decrement a count by a given value (defaults to 1)
+    def decr( stat, by=1 )
+      redis.decrby("stat:#{stat}", by)
+    end
+
+    # reset this stat to 0
+    def reset( stat )
+      redis.del( "stat:#{stat}" )
+    end
+
+  end
+end

data/lib/kthxbye/version.rb

@@ -1,5 +1,5 @@
 module Kthxbye
   # Returns current version of Kthxbye
-  Version = VERSION = "1.2.1"
+  Version = VERSION = "1.3.0"
 end
 

data/lib/kthxbye/web_interface.rb

@@ -99,6 +99,11 @@ module Kthxbye
       redirect url(:overview)
     end
 
+    get "/queues/:q" do
+      @queue = params[:q]
+      haml :queue, {:layout => :layout} 
+    end
+
     post "/failed/:job/rerun" do
       Kthxbye::Failure.rerun(params[:job])
       redirect url(:failed)
@@ -109,6 +114,11 @@ module Kthxbye
       redirect url(:failed)
     end
 
+    post "/failed/clear" do
+      Kthxbye::Failure.clear_all
+      redirect url(:queues)
+    end
+
     post "/toggle_polling" do
       redirect url(:overview)
     end

data/lib/kthxbye/web_interface/public/style.css

@@ -5,6 +5,11 @@ body {
 	font-family: Helvetica, Arial, sans-serif;
 }
 
+h2 {
+  margin-bottom: 0;
+  color:#5d6f72;
+}
+
 #header {
 	/*height: 50px;*/
 	background-color: #b5d3d5;
@@ -37,7 +42,6 @@ body {
 }
 
 a {
-	text-decoration: none;
 	color: #619793;
 }
 
@@ -126,3 +130,8 @@ table tr td.result, table tr th.result {
 #backtrace {
   padding:20px;
 }
+
+.t8 {
+  font-size: 8pt;
+  color: #5e5f5f;
+}

data/lib/kthxbye/web_interface/views/failed.haml

@@ -17,7 +17,7 @@
           %a{:href => "/view_backtrace?id=#{f['job']}"}
             View Backtrace
         %td #{f['time']}
-        %td
+        %td{:style => "white-space:nowrap;"}
           %form{:action => "/failed/#{f['job']}/rerun", :method => "post", :style => "display:inline"}
             %input{:class => "awesome button small", :value => "Retry", :type => "submit"}
           %span.spacer

data/lib/kthxbye/web_interface/views/queue.haml

@@ -0,0 +1,14 @@
+%h2 The #{params[:q]} queue
+%table.queue_job
+  %tr
+    %th.id ID
+    %th.klass Class
+    %th.payload Data
+    %th.klass Status
+  - Kthxbye.data_peek(@queue).each do |id,job|
+    %tr
+      %td.id #{id}
+      %td.klass #{job['klass']}
+      %td.payload #{job['payload']}
+      %td.klass #{Kthxbye::Job.find(id,@queue).status}
+

data/lib/kthxbye/web_interface/views/queues.haml

@@ -1,31 +1,30 @@
-%h2 Queues:
+%h2 Queues
+.t8 The queue job count only shows jobs that are either waiting for processing or being processed
 
-- kthxbye.queues.each do |queue|
-  #queue{:style => "margin-top:20px"}
-    #queue_head
-      Pending jobs on 
-      %b #{queue} 
-      (#{Kthxbye.size(queue)})
-      %span.spacer
-        |
-      %form{:action => "/queues/#{queue}/remove", :style => "display:inline", :method => "post" }
-        %input{:type => "submit", :class => "awesome button small", :value => "Remove Queue"}
-          
-
-    %br
-
-    %table.queue_job
-      %tr
-        %th.id ID
-        %th.klass Class
-        %th.payload Data
-        %th.klass Status
-      - Kthxbye.data_peek(queue).each do |id,job|
-        %tr
-          %td.id #{id}
-          %td.klass #{job['klass']}
-          %td.payload #{job['payload']}
-          %td.klass #{Kthxbye::Job.find(id,queue).status}
+%br
 
+%table
+  %tr
+    %th Name
+    %th Jobs
+    %th  
+  - kthxbye.queues.each do |queue|
+    %tr
+      %td
+        %a{:href => "/queues/#{queue}"}
+          %b #{queue} 
+      %td
+        = Kthxbye.size(queue)
+      %td
+        %form{:action => "/queues/#{queue}/remove", :style => "display:inline", :method => "post" }
+          %input{:type => "submit", :class => "awesome button small", :value => "Remove Queue"}
+  %tr{:style => "background-color:#F4A8AA"}
+    %td 
+      %a{:href => "/failed"}
+        %b Failed
+    %td= Kthxbye::Failure.count
+    %td
+      %form{:action => "/failed/clear", :style => "display:inline", :method => "post" }
+        %input{:type => "submit", :class => "awesome button small", :value => "Clear"}
 
 

data/lib/kthxbye/web_interface/views/stats.haml

@@ -1,7 +1,7 @@
 - @sub_tabs = %w(kthxbye redis keys)
 
 %h2 
-  Stats about #{params[:id] || params[:key]}
+  Stats for #{params[:id] || params[:key] || "kthxbye"}
 
 %p
   - if params[:key] 

data/lib/kthxbye/web_interface/views/workers.haml

@@ -5,16 +5,25 @@
     %th PID
     %th Server
     %th Queues
+    %th Uptime
+    %th Jobs Run
     %th Working Job
     - kthxbye.workers.each do |worker|
       - host,pid,queues = worker.to_s.split(":")
       %tr
         %td #{pid}
         %td #{host}
-        %td #{queues}
+        %td
+          - queues.split(",").each do |queue|
+            %a{:href => "/queues/#{queue}"}
+              #{queues}
+        %td
+          = Time.parse(worker.started).strftime("%m/%d/%Y at %I:%M%p") if worker.started
+        %td
+          = worker.processed
         %td 
           - job = Kthxbye::Worker.working_on(worker)
-          = job ? "<b>ID</b>: #{job['job_id']} -  <b>Started</b>: #{job['started']}" : "***"
+          = job ? "<b>ID</b>: #{job['job_id']} -  <b>Started</b>: #{Time.parse(job['started']).strftime("%I:%M%p")}" : "***"
 
 - else
   %span{:style => "color:red"}

data/lib/kthxbye/web_interface/views/working.haml

@@ -1,4 +1,6 @@
-%h2 Active Workers
+%h2 #{Kthxbye.working.size} of #{Kthxbye.workers.size} workers active:
+
+%br
 
 - unless Kthxbye.working.empty?
   %table
@@ -11,7 +13,7 @@
         %td #{job['job_id']}
         %td #{job['started']}
 - else
-  %span{:style => "color:red"}
+  %span{:style => "color:#A6373A"}
     Currently no active workers.
 
 %p

data/lib/kthxbye/worker.rb

@@ -69,8 +69,7 @@ module Kthxbye
 
           @child = fork {
             log "Forking..."
-            result = job.perform
-            yield job if block_given?
+            perform(job)
             exit!
           }
 
@@ -86,6 +85,19 @@ module Kthxbye
       unregister_worker
     end
 
+    def perform(job)
+      begin
+        result = job.perform
+      rescue Object => ex
+        job.fail(ex)
+        log "Error running job #{job}"
+        Stats.incr("failures:#{self}")
+        perform(job) if job.failed_attempts < Kthxbye::Config.options[:attempts]
+      ensure
+        yield job if block_given?
+      end
+    end
+
     # Returns the queues this worker is attached toin alphabetical order.
     def queues
       @queues.sort
@@ -117,6 +129,7 @@ module Kthxbye
     def register_worker
       log "Registered worker #{self}"
       redis.sadd( :workers, self ) if !exists?
+      redis.set( "worker:#{self}:started", Time.now.to_s )
     end
 
     # Removes the worker from our worker registry
@@ -130,6 +143,10 @@ module Kthxbye
 
       redis.del "worker:#{self}"
       redis.srem :workers, self
+      redis.del "worker:#{self}:started"
+
+      Stats.reset("processed:#{self}")
+      Stats.reset("failures:#{self}")
     end
     
     # Gets the current job this worker is working.
@@ -163,9 +180,23 @@ module Kthxbye
       redis.del( "worker:#{self}" )
       log "Completed job #{@current_job}"
       redis.publish("job.completed", @current_job.id)
+      Stats.incr("processed")
+      Stats.incr("processed:#{self}")
       @current_job = nil
     end
 
+    def processed
+      Stats["processed:#{self}"]
+    end
+
+    def failed
+      Stats["failures:#{self}"]
+    end
+
+    def started
+      redis.get( "worker:#{self}:started" )
+    end
+
     #
     # thanks to http://github.com/defunkt/resque/blob/master/lib/resque/worker.rb for these signals
     #

data/test/test_kthxbye.rb

@@ -123,12 +123,12 @@ class TestKthxbye < Test::Unit::TestCase
 
     should "retry a job if attempts setting is > 1" do
       Kthxbye::Config.options[:attempts] = 2
+      worker = Kthxbye::Worker.new("test", 0)
       id = Kthxbye.enqueue( "test", BadJob )
-      job = Kthxbye::Job.find(id, "test")
 
-      job.perform
-      assert_equal 2, job.instance_variable_get(:@failed_attempts)
-      
+      worker.run
+      job = Kthxbye::Job.find(id, "test")
+      assert_equal 2, job.failed_attempts
     end
 
     should "delete a job" do
@@ -191,8 +191,8 @@ class TestKthxbye < Test::Unit::TestCase
       worker1.run
       assert_equal :succeeded, job.status
 
-      job = Kthxbye::Job.find(bad_job, "bad")
       worker2.run
+      job = Kthxbye::Job.find(bad_job, "bad")
 
       assert_equal :failed, job.status
     end

data/test/test_worker.rb

@@ -2,7 +2,7 @@
 class TestWorker < Test::Unit::TestCase
   context "See Kthxbye::Worker" do
     setup do
-      Kthxbye::Config.setup(:verbose => false, :redis_port => 9876) 
+      Kthxbye::Config.setup(:verbose => false, :redis_port => 9876, :attempts => 1) 
       Kthxbye.redis.flushall
       
       @id = Kthxbye.enqueue( "test", SimpleJob, 1, 2 )
@@ -130,6 +130,29 @@ class TestWorker < Test::Unit::TestCase
       end
     end
 
+    should "report system stats" do
+      Kthxbye.enqueue( "test", BadJob )
+      assert_equal 0, @worker.processed
+      assert_equal 0, @worker.failed
+
+      @worker.run
+
+      assert_equal 2, Kthxbye::Stats["processed"]
+      assert_equal 1, Kthxbye::Stats["failures"]
+    end
+
+    should "track failures" do
+      Kthxbye.enqueue( "test", BadJob )
+      Kthxbye.enqueue( "test", BadJob )
+
+      3.times do
+        job = @worker.grab_job
+        @worker.perform(job)
+      end
+
+      assert_equal 2, @worker.failed
+    end
+
     should "requeue a job and report error if worker is killed mid-process" do
       @worker.working( Kthxbye::Job.find( @id, "test" ) )
       @worker.unregister_worker

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kthxbye
 version: !ruby/object:Gem::Version 
+  hash: 27
   prerelease: false
   segments: 
   - 1
-  - 2
-  - 1
-  version: 1.2.1
+  - 3
+  - 0
+  version: 1.3.0
 platform: ruby
 authors: 
 - Luke van der Hoeven
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-08 00:00:00 -04:00
+date: 2010-10-11 00:00:00 -04:00
 default_executable: kthxbye-monitor
 dependencies: []
 
@@ -42,6 +43,7 @@ files:
 - lib/kthxbye/web_interface/views/overview.haml
 - lib/kthxbye/web_interface/views/failed.haml
 - lib/kthxbye/web_interface/views/set.haml
+- lib/kthxbye/web_interface/views/queue.haml
 - lib/kthxbye/web_interface/views/view_backtrace.haml
 - lib/kthxbye/web_interface/views/layout.haml
 - lib/kthxbye/web_interface/views/queues.haml
@@ -51,6 +53,7 @@ files:
 - lib/kthxbye/failure.rb
 - lib/kthxbye/config.rb
 - lib/kthxbye/railtie.rb
+- lib/kthxbye/stats.rb
 - lib/kthxbye/helper.rb
 - lib/kthxbye/worker.rb
 - lib/kthxbye/railties/kthxbye.rake
@@ -85,6 +88,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
@@ -93,6 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 23
       segments: 
       - 1
       - 3