kthxbye 1.0.0

data/lib/kthxbye/config.rb

@@ -0,0 +1,35 @@
+module Kthxbye
+  module Config
+
+    # default options for Kthxbye
+    #
+    # redis_server = the ip to connect to by defaut
+    # 
+    # redis_port = default redis port
+    # 
+    # attempts = default number of attempts on a failing job
+    # before moving to the failed job store
+    #
+    # vervose = more output 
+    #
+    DEFAULT = {:redis_server => '127.0.0.1',
+      :redis_port => 9876,
+      :attempts => 1,
+      :verbose => false}.freeze
+    
+    # configures any other args input by the user. 
+    # can pull from a config.yaml file as well.
+    #
+    def self.setup( args=nil )
+      @options = DEFAULT.dup
+      @options.merge!( YAML.load('config.yaml') ) if File.exist?( 'config.yaml' )
+      @options.merge!( args ) if args
+    end
+
+    def self.options
+      return @options if @options
+      Config.setup
+    end
+
+  end
+end

data/lib/kthxbye/exceptions.rb

@@ -0,0 +1,4 @@
+module Kthxbye
+  # raised when a worker is killed during processing
+  class ActiveWorkerKilled < RuntimeError; end
+end

data/lib/kthxbye/failure.rb

@@ -0,0 +1,62 @@
+module Kthxbye  
+  module Failure
+    include Helper
+    extend Helper
+
+    def self.all
+      redis.hvals( :failed ).sort.map{|x| decode( x )}
+    end
+
+    # returns a specified failed job data
+    def self.find(id)
+      decode( redis.hget( :failed, id ) )
+    end
+
+    # gets count of all errors
+    def self.count
+      redis.hkeys( :failed ).size
+    end
+
+    # gets count of all errors of a specific type
+    def self.count_type(type)
+      vals = redis.hvals( :failed )
+      vals.each {|x| o = decode(x); vals.delete x if o['type'] !~ /#{type.to_s}/}
+      vals.size
+    end
+
+    # creates a Failure object.
+    def self.create(job, exception)
+      failed_attempts = (Failure.find(job.id)['attempts'].to_i + 1) if redis.hexists( :failed, job.id )
+
+      error = {
+        :type => exception.class.to_s,
+        :error => exception.to_s,
+        :job => job.id,
+        :queue => job.queue,
+        :time => Time.now,
+        :backtrace => Array( exception.backtrace ),
+        :attempts => (failed_attempts || 1)
+      }
+  
+      redis.hset( :failed, job.id, encode( error ) )
+
+      job.dequeue
+    end
+
+    def self.rerun(id)
+      failure = Failure.find(id)
+      Job.find(id, failure['queue']).rerun
+    end
+
+    def self.fails_for_job(id)
+      failure = decode( redis.hget( :failed, id ) )
+      return failure ? failure['attempts'] : 0
+    end
+
+    # the only method allowed to clear exceptions out of the exception store
+    def self.clear_exception(id)
+      redis.hdel( :failed, id ) 
+    end
+
+  end
+end

data/lib/kthxbye/helper.rb

@@ -0,0 +1,42 @@
+module Kthxbye
+  module Helper
+
+    def redis
+      Kthxbye.redis
+    end
+
+    def log(msg)
+      if Kthxbye::Config.options[:verbose]
+        puts "!! #{msg} - #{Time.now.strftime("%I:%M%p")}"
+      end
+    end
+
+    #
+    # encode/decode code taken and modified from Resque 
+    # (http://github.com/defunkt/resque/blob/master/lib/resque/helpers.rb)
+    # 
+    def encode( data )
+      if defined? Yajl
+        Yajl::Encoder.encode(data)
+      else
+        data.to_json
+      end
+    end
+
+    def decode( data )
+      return unless data
+
+      if defined? Yajl
+        begin
+          Yajl::Parser.parse( data, :check_utf8 => false )
+        rescue Yajl::ParseError
+        end
+      else
+        begin
+          JSON.parse( data )
+        rescue JSON::ParseError
+        end
+      end
+    end
+  end
+end

data/lib/kthxbye/job.rb

@@ -0,0 +1,127 @@
+module Kthxbye
+  class Job
+    include Helper
+    extend Helper
+
+    attr_accessor :id, :queue, :data, :worker
+    attr_reader :failed_attempts
+
+    def self.add_to_queue(queue, id)
+      redis.rpush( "queue:#{queue}", id )
+    end
+
+    # insert a job into the queue
+    def self.create(queue, klass, *args)
+      raise "Need a queue to store job in" if queue.to_s.empty?
+      raise "No class to reference job type by" if klass.nil?
+
+      redis.incr :uniq_id
+      id = redis.get :uniq_id
+
+      Job.add_to_queue( queue, id )
+      Kthxbye.register_queue( queue )
+      
+      redis.hset( "data-store:#{queue}", id, encode( {:klass => klass, :payload => args} ) )
+      log "Created job in queue #{queue} with an unique key of #{id}"
+
+      return id.to_i
+    end
+
+    def self.find(id, queue)
+      data = decode( redis.hget( "data-store:#{queue}", id ) )
+      data ? Job.new(id, queue, data) : nil
+    end
+    
+    # removes all existence of this job and its data
+    # returns the last known status of the job
+    def self.destroy(id, queue)
+      ret = Job.find(id, queue).status
+
+      # remove the element from the active queue
+      redis.lrem("queue:#{queue}", 0, id) 
+      # be sure we also remove it from the inactive queue
+      redis.srem("queue:#{queue}:inactive", id)
+      # remove the job's data as well
+      redis.hdel("data-store:#{queue}", id) 
+      redis.hdel("result-store:#{queue}", id)
+      redis.hdel( :faulure, id )
+      
+      return ret
+    end
+
+    # instantiates a job for the worker to run
+    def initialize(id, queue, data)
+      @id = id.to_i
+      @queue = queue
+      @data = data
+      @failed_attempts =  Failure.fails_for_job(@id) # local tracking only, for rerun purposes
+    end
+
+    # simply requeues a job
+    def rerun
+      Job.add_to_queue( @queue, @id )
+    end
+
+    def status
+      if result
+        :succeeded
+      elsif Failure.find(@id)
+        :failed
+      elsif active?
+        :active
+      else
+        :inactive
+      end
+    end
+
+    def result
+      decode( redis.hget("result-store:#{@queue}", @id) )
+    end
+
+    # simply removes this job from the active queue and places it
+    # on the inactive list.
+    # does not remove job payload
+    def dequeue
+      redis.lrem("queue:#{@queue}", 0, @id) 
+      inactive
+    end
+  
+    # does the heavy lifting of running a job
+    def perform
+      begin
+        @klass = Object.const_get(@data['klass'])
+        @payload = @data['payload']
+
+        result = @klass.send(:perform, *@payload)
+        redis.hset( "result-store:#{@queue}", @id, encode( result ) )
+        return result
+      rescue Exception => ex
+        @failed_attempts += 1
+        log "Error occured: #{ex.message}. Try: #{@failed_attempts}/#{Kthxbye::Config.options[:attempts]}"
+        return Kthxbye::Failure.create( self, ex ) if @failed_attempts >= Kthxbye::Config.options[:attempts]
+        perform
+      end
+    end
+
+    # will allow us to track when this job is being worked
+    def active
+      redis.srem("jobs:inactive", @id)
+    end
+
+    def active?
+      !redis.sismember("jobs:inactive", @id)
+    end
+
+    def inactive
+      redis.sadd("jobs:inactive", @id)
+    end
+
+    def ==(obj)
+      return false if obj.nil?
+      @data == obj.data &&
+        @id == obj.id &&
+        @queue == obj.queue
+    end
+
+  end
+end

data/lib/kthxbye/version.rb

@@ -0,0 +1,5 @@
+module Kthxbye
+  # may be a bad idea if only lib dir is installed...
+  Version = VERSION = File.read(File.dirname(__FILE__)+'/../../VERSION').chomp
+end
+

data/lib/kthxbye/web_interface.rb

@@ -0,0 +1,117 @@
+require 'sinatra/base'
+require 'haml'
+require 'kthxbye'
+
+module Kthxbye
+  class WebInterface < Sinatra::Base
+    base = File.dirname( File.expand_path(__FILE__) )
+
+    set :views, "#{base}/web_interface/views"
+    set :public, "#{base}/web_interface/public"
+    set :static, true
+    set :environment, :development
+
+    helpers do
+      include Rack::Utils
+
+      def toggle_poll
+        if @active
+          text = "Last update: #{Time.now.strftime("%H:%M:%S")}"
+        else
+          text = "<a rel='poll_link' class='awesome button small' href='#{url(request.path_info)}.poll'>Live Poll</a>"
+        end
+        "<div id='poll_status'>#{text}</div>"
+      end
+
+      def url(*path_parts)
+        [ path_prefix, path_parts ].join("/").squeeze('/')
+      end
+
+      def path_prefix
+        request.env['SCRIPT_NAME']
+      end
+
+      def show(page, layout=true)
+        begin
+          haml page.to_sym, {:layout => layout}, :kthxbye => Kthxbye
+        rescue Errno::ECONNREFUSED
+          haml :error, {:layout => false}, :error => "Can't connect to Redis at #{Kthxbye::Config.options[:redis_server]}:#{Kthxbye::Config.options[:redis_port]}"
+        end
+      end
+      
+      def partial(template, local_vars = {})
+        haml template.to_sym, {:layout => false}, local_vars
+      end
+      
+      def current_url
+        url request.path_info.split("/")
+      end
+
+      def mark_current(page)
+        'current' if current_url[1..-1] == page.split(" ").first
+      end
+
+      def tab(name)
+        name.to_s.downcase!
+        path = url(name)
+        "<a href=#{path}><span class='tab #{mark_current(name)}'>#{name}</span></a>"
+      end
+
+      def tabs
+        @tabs ||= ["Overview", "Workers (#{Kthxbye.workers.size})", "Failed (#{Kthxbye::Failure.all.size})", "Queues (#{Kthxbye.queues.size})", "Stats"]
+      end
+    end
+    
+    # default route
+    get "/?" do
+      redirect url(:overview)
+    end
+
+    %w(overview workers failed queues stats).each do |page|
+      get "/#{page}" do
+        show page
+      end
+    end
+
+    get "/stats/:id" do
+      show :stats 
+    end
+
+    get "/stats/keys/:key" do
+      show :stats
+    end
+
+    %w(overview workers).each do |page|
+      get "/#{page}.poll" do
+        @active = true
+        content_type "text/plain"
+        show(page.to_sym, false)
+      end
+    end
+
+    get "/view_backtrace" do
+      @backtrace = Kthxbye::Failure.find(params[:id])['backtrace']
+      haml :view_backtrace, {:layout => :layout}
+    end
+
+    post "/queues/:q/remove" do
+      Kthxbye.unregister_queue(params[:q])
+      redirect url(:overview)
+    end
+
+    post "/failed/:job/rerun" do
+      Kthxbye::Failure.rerun(params[:job])
+      redirect url(:failed)
+    end
+    
+    post "/failed/:job/clear" do
+      job = Kthxbye::Failure.clear_exception(params[:job])
+      redirect url(:failed)
+    end
+
+    post "/toggle_polling" do
+      redirect url(:overview)
+    end
+
+  end
+end

data/lib/kthxbye/web_interface/public/application.js

@@ -0,0 +1,16 @@
+$(function() {
+  var poll_interval = 2
+
+  $('a[rel="poll_link"]').click(function(e) {
+    e.preventDefault()
+    var action = this.href
+
+    $('div#poll_status').html("Starting...")
+    
+    setInterval(function() {
+      $("#content").load(action)
+    }, 1000 * poll_interval)
+    
+    return false
+  })
+})

data/lib/kthxbye/web_interface/public/awesome-buttons.css

@@ -0,0 +1,108 @@
+/*
+    awesome buttons are based on a blog post by ZERB
+    Read there blog post for more information:
+    "Super awesome Buttons with CSS3 and RGBA":http://www.zurb.com/article/266/super-awesome-buttons-with-css3-and-rgba
+
+    this buttons are even more awesome, as the need only one color for all three states,
+    and have an super awesome onclick state
+*/
+
+/* set an awesome color for the buttons, feel free to add new colors like an .awesome.green or .awesome.secondary */
+.awesome {
+  background-color: #5D6F72 !important;
+  color: #fff !important;
+}
+
+/* the awesome size gets set here. Feel free to add new sizes, like .awesome.small or .small.large */
+.awesome { padding: 5px 10px 6px !important; font-size: 13px !important; }
+.awesome:active { padding: 6px 10px 5px !important; }
+
+.awesome.small { padding: 2px, 5px, 3px !important; font-size: 10px !important; }
+
+/* Touch the rest at your onw risk. */
+.awesome {
+  border: 0 !important;
+  cursor: pointer !important;
+  font-style: normal !important;
+  font-weight: bold !important;
+  font: inherit !important;
+  line-height: 1 !important;
+  position: relative !important;
+  text-align: center !important;
+  text-decoration: none !important;
+
+  /* vertical margin is the opposite of button's awesomeness */
+  margin-top: 0 !important;
+  margin-bottom: 0 !important;
+
+  /* not all browser support these, but who cares? */
+  text-shadow: 0 -1px 1px rgba(0,0,0,0.25), -2px 0 1px rgba(0,0,0,0.25) !important;
+
+          border-radius: 5px !important;
+     -moz-border-radius: 5px !important;
+  -webkit-border-radius: 5px !important;
+          box-shadow: 0 1px 2px rgba(0,0,0,0.5) !important;
+     -moz-box-shadow: 0 1px 2px rgba(0,0,0,0.5) !important;
+  -webkit-box-shadow: 0 1px 2px rgba(0,0,0,0.5) !important;
+
+  /* who needs images these days? */
+  background-image: -moz-linear-gradient(top, rgba(255,255,255,.2), rgba(150,150,150,.2), rgba(0,0,0,.0)) !important;
+  background-image: -webkit-gradient(linear, 0% 0%, 0% 100%, from(rgba(255,255,255,.2)), color-stop(0.5, rgba(150,150,150,.2)), to(rgba(0,0,0,.0))) !important;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr=#30ffffff,endColorstr=#10000000) progid:DXImageTransform.Microsoft.Shadow(color=#000000,direction=135,strength=2);
+
+  /* cross browser inline block hack
+     see http://blog.mozilla.com/webdev/2009/02/20/cross-browser-inline-block/ */
+  display: -moz-inline-stack;
+  display: inline-block;
+  vertical-align: middle !important;
+  *display: inline !important;
+  position: relative;
+
+  /* IE luv */
+  zoom: 1;
+
+  /* disable text selection (Firefox only) */
+  -moz-user-select: none;
+}
+
+/* OPERA only
+   if there is no border set, Opera shows a transparent background-color if border-radius is set. */
+@media all and (-webkit-min-device-pixel-ratio:10000),not all and (-webkit-min-device-pixel-ratio:0) {
+  input.awesome {
+    border: 1px solid RGBa(0,0,0,0) !important;
+  }
+}
+
+/* hide selection background color */
+.awesome::selection {
+  background: transparent;
+}
+
+.awesome {
+  outline: 0; /* remove dotted line, works for links only */
+}
+.awesome::-moz-focus-inner {
+  border: none; /* remove dotted lines for buttons */
+}
+.awesome:focus,
+.awesome:hover {
+  background-image: -moz-linear-gradient(top, rgba(255,255,255,.4), rgba(150,150,150,.3), rgba(0,0,0,.0)) !important;
+  background-image: -webkit-gradient(linear, 0% 0%, 0% 100%, from(rgba(255,255,255,.4)), color-stop(0.5, rgba(150,150,150,.3)), to(rgba(0,0,0,.0))) !important;
+  #filter: progid:DXImageTransform.Microsoft.gradient(startColorstr=#50ffffff,endColorstr=#20000000) progid:DXImageTransform.Microsoft.Shadow(color=#000000,direction=135,strength=2);
+}
+.awesome:active {
+  background-image: -moz-linear-gradient(top, rgba(0,0,0,.2), rgba(150,150,150,.2), rgba(255,255,255,.2)) !important;
+  background-image: -webkit-gradient(linear, 0% 0%, 0% 100%, from(rgba(0,0,0,.2)), color-stop(0.5, rgba(150,150,150,.2)), to(rgba(255,255,255,.2))) !important;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr=#20000000,endColorstr=#50ffffff) progid:DXImageTransform.Microsoft.Shadow(color=#000000,direction=315,strength=1);
+
+          box-shadow: inset 0 1px 2px rgba(0,0,0,0.7) !important;
+     -moz-box-shadow: inset 0 1px 2px rgba(0,0,0,0.7) !important;
+  -webkit-box-shadow: inset 0 1px 2px rgba(0,0,0,0.7) !important;
+}
+
+/* Safari doesn't support inset box shadow, so we better remove it */
+@media screen and (-webkit-min-device-pixel-ratio:0){
+  .awesome:active {
+    -webkit-box-shadow: none;
+  }
+}