steve 1.0.0

data/lib/steve/interface/public/style.css

@@ -0,0 +1,45 @@
+html { color: #000; background: #FFF; }
+body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,textarea,p,blockquote,th,td { margin: 0; padding: 0; }
+li { list-style: none; }
+h1, h2, h3, h4, h5, h6 { font-size: 100%; font-weight: normal; }
+pre, form { font-style: normal; font-weight: normal; }
+fieldset { border: 0; }
+legend { color: #000; }
+input, textarea { margin: 0; padding: 0; font-family: inherit; font-size: inherit; font-weight: inherit; *font-size: 100%; }
+p, blockquote { margin: 0; padding: 0; }
+th { margin: 0; padding: 0; font-style: normal; font-weight: normal; text-align: left; }
+table { border-collapse: collapse; border-spacing: 0; }
+img { border: 0; }
+address { font-style: normal; font-weight: normal; }
+caption { font-style: normal; font-weight: normal; text-align: left; }
+cite, dfn, em, strong, var { font-style: normal; font-weight: normal; }
+q:before, q:after { content: ''; }
+abbr, acronym { border: 0; font-variant: normal; }
+sup { vertical-align: text-top; }
+sub { vertical-align: text-bottom; }
+select { font-family: inherit; font-size: inherit; font-weight: inherit; *font-size: 100%; }
+
+
+html { font-family:"Helvetica Neue", Arial, sans-serif; font-size:13px; margin:0; }
+body { margin:0;}
+#header { background:#000; color:#fff; padding:10px 2%;}
+#header h1 { color:#999;}
+#header h1 span { font-weight:bold; font-size:90%; color:#fff;}
+#header h1 em { color:yellow; font-weight:bold;}
+
+#content { margin:15px 2%;}
+#content h2 { font-size:150%; font-weight:bold; margin:15px 0;}
+#content table.data {width:100%;}
+#content table.data thead td { background:#efefef; font-weight:bold; font-size:90%;}
+#content table.data td { padding:5px; border:1px solid #ccc;}
+#content table.data td code span { display:block; font-size:70%;}
+#content table.data td.none { color:#999; font-style:italic; font-size:80%;}
+#content ul { line-height:1.5; }
+#content ul li { list-style:square; margin-left:25px;}
+#content p { margin:10px 0; line-height:1.5;}
+#content a { color:#333;}
+
+#content dl {margin:40px 0; font-size:110%;}
+#content dl dt { font-weight:bold; float:left; width:150px; text-align:right;}
+#content dl dd { margin-left:180px; margin-bottom:10px;}
+#content dl dd pre { background:#000; color:#fff; line-height:1.3;  padding:6px;}

data/lib/steve/interface/views/completed.haml

@@ -0,0 +1,41 @@
+!!!
+%html
+  %head
+    %title Steve - Job Queue Monitor
+    %link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
+  %body
+    #header
+      %h1 <span>Steve</span>&mdash;Job Queue Monitor
+
+    #content
+      #recent
+        %h2 Recently completed jobs
+        %table.data
+          %thead
+            %tr
+              %td{:width => '35%'} Job
+              %td{:width => '25%'} Object
+              %td{:width => '20%'} Queued at
+              %td{:width => '15%'} Run at
+              %td{:width => '5%'} Retries
+          %tbody
+            - jobs = Steve::QueuedJob.completed.limit(15).order('run_at desc')
+            - for job in jobs
+              %tr
+                %td
+                  %code
+                    %a{:href => "view?id=#{job.id}"}= job.job
+                    %span= job.params.inspect
+                %td
+                  - if job.associated_object_type.blank?
+                    &mdash;
+                  - else
+                    == #{job.associated_object_type}##{job.associated_object_id}
+                %td= job.created_at.to_s(:db)
+                %td= job.run_at.to_s(:db)
+                %td= job.retries
+            - if jobs.empty?
+              %tr
+                %td.none{:colspan => 7} There are no jobs recently completed
+      %p
+        %a{:href => "/jobs"} Back to overview

data/lib/steve/interface/views/failed.haml

@@ -0,0 +1,43 @@
+!!!
+%html
+  %head
+    %title Steve - Job Queue Monitor
+    %link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
+  %body
+    #header
+      %h1 <span>Steve</span>&mdash;Job Queue Monitor
+
+    #content
+      %h2 Failed jobs
+      %p
+        The jobs listed below have failed to execute recently. You can clear all the jobs using the button below, this will
+        remove them from the database and you will not be able to retry once removed.
+      %table.data
+        %thead
+          %tr
+            %td{:width => '15%'} Date
+            %td{:width => '5%'} ID
+            %td{:width => '35%'} Job
+            %td{:width => '10%'} Queue
+            %td{:width => '35%'} Error
+            %td{:width => '10%'}
+        %tbody
+          - jobs = Steve::QueuedJob.failed
+          - for job in jobs
+            %tr
+              %td= job.finished_at.to_s(:short)
+              %td= job.id
+              %td
+                %code
+                  %a{:href => "view?id=#{job.id}"}= job.job
+                  %span= job.params.inspect
+              %td= job.queue
+              %td
+                %code{:style => 'font-size:90%', :title => job.error}= ERB::Util.html_escape(job.error.split("\n").first)\
+              %td
+                %a{:href => '#'} Retry
+          - if jobs.empty?
+            %tr
+              %td.none{:colspan => 6} There are no jobs currently running
+      %p
+        %a{:href => "/jobs"} Back to overview

data/lib/steve/interface/views/index.haml

@@ -0,0 +1,74 @@
+!!!
+%html
+  %head
+    %title Steve - Job Queue Monitor
+    %link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
+  %body
+    #header
+      %h1 <span>Steve</span>&mdash;Job Queue Monitor
+
+    #content
+      #active
+        %h2 Jobs being executed at the moment
+        %table.data
+          %thead
+            %tr
+              %td{:width => '25%'} Job
+              %td{:width => '25%'} Queue
+              %td{:width => '50%'} Worker
+          %tbody
+            - jobs = Steve::QueuedJob.running
+            - for job in jobs
+              %tr
+                %td
+                  %code
+                    %b= job.job
+                    %span= job.params.inspect
+                %td= job.queue
+                %td= job.worker
+            - if jobs.empty?
+              %tr
+                %td.none{:colspan => 3} There are no jobs currently running
+  
+      #queues
+        %h2 Jobs which are pending execution
+        %table.data
+          %thead
+            %tr
+              %td{:width => '25%'} Job
+              %td{:width => '15%'} Queue
+              %td{:width => '10%'} Status
+              %td{:width => '10%'} Priority
+              %td{:width => '20%'} Queued at
+              %td{:width => '15%'} Run at
+              %td{:width => '5%'} Retries
+          %tbody
+            - jobs = Steve::QueuedJob.pending
+            - for job in jobs
+              %tr
+                %td
+                  %code
+                    %b= job.job
+                    %span= job.params.inspect
+                %td= job.queue
+                %td= job.status
+                %td= job.priority
+                %td= job.created_at.to_s(:db)
+                %td= job.run_at.to_s(:db)
+                %td= job.retries
+            - if jobs.empty?
+              %tr
+                %td.none{:colspan => 7} There are no jobs pending
+      
+        
+        
+      #stats
+        %h2 Statistics
+        %ul
+          %li== #{Steve::QueuedJob.count} jobs stored
+          %li== #{Steve::QueuedJob.pending.count} pending execution
+          %li== #{Steve::QueuedJob.running.count} running now
+          %li
+            %a{:href => 'jobs/completed'}== #{Steve::QueuedJob.completed.count} completed successfully
+          %li
+            %a{:href => 'jobs/failed'}== #{Steve::QueuedJob.failed.count} failed to execute

data/lib/steve/interface/views/object.haml

@@ -0,0 +1,38 @@
+!!!
+%html
+  %head
+    %title Steve - Job Queue Monitor
+    %link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
+  %body
+    #header
+      %h1 <span>Steve</span>&mdash;Job Queue Monitor
+
+    #content
+      #recent
+        %h2 Jobs for #{@req.params['type']}##{@req.params['id']}
+        %table.data
+          %thead
+            %tr
+              %td{:width => '35%'} Job
+              %td{:width => '10%'} Status
+              %td{:width => '15%'} Queue
+              %td{:width => '20%'} Queued at
+              %td{:width => '15%'} Run at
+              %td{:width => '5%'} Retries
+          %tbody
+            - for job in @jobs
+              %tr
+                %td
+                  %code
+                    %a{:href => "view?id=#{job.id}"}= job.job
+                    %span= job.params.inspect
+                %td= job.status
+                %td= job.queue
+                %td= job.created_at.to_s(:db)
+                %td= job.run_at.to_s(:db)
+                %td= job.retries
+            - if @jobs.empty?
+              %tr
+                %td.none{:colspan => 7} There are no jobs recently completed
+      %p
+        %a{:href => "/jobs"} Back to overview

data/lib/steve/interface/views/view.haml

@@ -0,0 +1,52 @@
+!!!
+%html
+  %head
+    %title Steve - Job Queue Monitor
+    %link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
+  %body
+    #header
+      %h1 <span>Steve</span>&mdash;Job Queue Monitor
+
+    #content
+      %dl
+        %dt Job
+        %dd= @job.job
+        
+        %dt Queue
+        %dd= @job.queue
+        
+        %dt Status
+        %dd= @job.status
+        
+        %dt Parameters
+        %dd= @job.params.inspect
+        
+        %dt Association?
+        %dd== #{@job.associated_object_type}##{@job.associated_object_id}
+        
+        %dt Run at
+        %dd= @job.run_at
+        
+        %dt Queued at
+        %dd= @job.created_at
+        
+        %dt Started at
+        %dd= @job.started_at
+        
+        %dt Finished at
+        %dd= @job.finished_at
+        
+        - unless @job.output.blank?
+          %dt Output
+          %dd
+            %pre~ preserve(@job.output)
+
+        - unless @job.error.blank?
+          %dt Error
+          %dd
+            %pre~ preserve(@job.error || "&nbsp;")
+        
+        %dt Worker
+        %dd= @job.worker
+      %p
+        %a{:href => "/jobs"} Back to overview

data/lib/steve/interface.rb

@@ -0,0 +1,48 @@
+require 'haml'
+
+module Steve
+  class Interface
+    
+    def call(env)
+      @req = Rack::Request.new(env)
+      case env['PATH_INFO'].to_s
+      when ''
+        [200, {'Content-type' => 'text/html'}, [haml(:index)]]
+      when /failed/
+        [200, {'Content-type' => 'text/html'}, [haml(:failed)]]
+      when /completed/
+        [200, {'Content-type' => 'text/html'}, [haml(:completed)]]
+      when /object/
+        @jobs = Steve::QueuedJob.where(:associated_object_type => @req.params['type'], :associated_object_id => @req.params['id']).order('created_at desc').limit(25)
+        [200, {'Content-type' => 'text/html'}, [haml(:object)]]
+      when /view/
+        @job = Steve::QueuedJob.find(@req.params['id'])
+        [200, {'Content-type' => 'text/html'}, [haml(:view)]]
+
+      else
+        path = static_path(env['PATH_INFO'])
+        if File.exist?(path)
+          [200, {}, [File.read(path)]]
+        else
+          [404, {'Content-type' => 'text/plain'}, ["Not found"]]
+        end
+        
+      end
+    end
+    
+    private
+    
+    def haml(view_name)
+      Haml::Engine.new(File.read(view_path(view_name))).render(self)
+    end
+    
+    def view_path(name)
+      File.expand_path(File.join('..', 'interface', 'views', name.to_s + '.haml'), __FILE__)
+    end
+    
+    def static_path(name)
+      File.expand_path(File.join('..', 'interface', 'public', name.to_s), __FILE__).gsub(/\.\./, '')
+    end
+    
+  end
+end

data/lib/steve/job.rb

@@ -0,0 +1,47 @@
+## Jobs can inherit from this class to add useful functionality and avoid
+## needing to specify an initializer for all your jobs.
+
+module Steve
+  class Job
+    
+    class Delay < StandardError; end
+    class Error < StandardError; end
+    
+    attr_reader :job
+    attr_reader :params
+    
+    def initialize(job, params = {})
+      @job    = job
+      @params = job.params
+    end
+    
+    ## Queue the job
+    def self.queue(params = {}, &block)
+      if Steve.run_jobs_in_foreground
+        Rails.logger.info "\e[33m  -> Started to exectute #{self.to_s} job in foreground\e[0m"
+        run(params)
+        Rails.logger.info "\e[33m  -> Finished executing #{self.to_s} job in foreground\e[0m"
+      else
+        Rails.logger.info "\e[33m  -> Queued #{self.to_s} job\e[0m"
+        QueuedJob.queue(self, params, &block)
+      end
+    end
+    
+    ## Live run a job after passing the params
+    def self.run(params = {})
+      self.new(FakeQueuedJob.new(params)).perform
+    end
+    
+    class FakeQueuedJob
+      attr_reader :params
+      def initialize(params)
+        @params = params
+      end
+      
+      def method_missing(*_)
+        true
+      end
+    end
+    
+  end
+end

data/lib/steve/queued_job.rb

@@ -0,0 +1,186 @@
+require 'stringio'
+
+module Steve
+  class QueuedJob < ActiveRecord::Base
+    
+    ## Set the table name - a migration should be created for this
+    set_table_name(Steve.jobs_table_name || 'jobs')
+    
+    ## Serialize the options
+    serialize :params
+    
+    ## Can belong to another active record object?
+    belongs_to :associated_object, :polymorphic => true
+    
+    ## Scopes
+    scope :pending, where(:status => ['pending', 'delayed'])
+    scope :running, where(:status => 'running')
+    scope :completed, where(:status => 'completed')
+    scope :failed, where(:status => 'failed')
+    
+    ## Automatically set defaults for new jobs
+    before_create do
+      self.priority ||= (Steve.default_job_priority || 10)
+      self.queue    ||= (Steve.default_job_queue || 'normal')
+      self.status   ||= 'pending'
+      self.run_at   ||= Time.now.utc
+    end
+    
+    ## Queue a new job for processing. Returns true or false depending whether
+    ## the job has been queued or not.
+    def self.queue(klass, params = {}, &block)
+      job = self.new
+      job.job = klass.to_s
+      job.params = params
+      job.queue = klass.instance_variable_get('@queue')
+      block.call(job) if block_given?
+      job.save
+    end
+    
+    ## Execute a new job from the queue. Returns true if a job was executed, or false if a 
+    ## job was not found or we couldn't obtain locks for them.
+    def self.execute_jobs(queue = '*', limit = 5)
+      pending_jobs = ActiveRecord::Base.silence do
+        jobs = self.where(:status => ['pending', 'delayed'], :worker => nil).where(["run_at <= ?", Time.now.utc]).order("priority asc").limit(5)
+        jobs = jobs.where(:queue => queue) unless queue.nil? or queue == '*'
+        jobs.all
+      end
+
+      jobs_executed = Array.new
+      for job in pending_jobs.sort_by { rand() }
+        Steve.log "[#{job.id}] Attempt to aquire lock"
+        if job.lock 
+          Steve.log "[#{job.id}] Lock acquired"
+          ActiveRecord::Base.remove_connection
+          if @child = fork
+            rand
+            Steve.log "[#{job.id}] Forked to #{@child}"
+            $0 = "sj: forked to #{@child} at #{Time.now.utc.to_s(:db)}"
+            ActiveRecord::Base.establish_connection
+            Process.wait
+          else
+            Steve.log "[#{job.id}] Executing"
+            $0 = "sj: executing job ##{job.id} since #{Time.now.utc.to_s(:db)}"
+            ActiveRecord::Base.establish_connection
+            Steve.after_job_fork.call if Steve.after_job_fork.is_a?(Proc)
+            job.execute
+            exit
+          end
+          jobs_executed << job
+        else
+          Steve.log "[#{job.id}] Lock could not be acquired. Moving on."
+        end
+      end
+      jobs_executed
+    end
+    
+    ## Execute this job, catching any errors if they occur and ensuring the
+    ## job is started & finished as appropriate.
+    def execute
+      @output_file = File.join('', 'tmp', "steve-job-#{self.id}")
+      job = self.job.constantize.new(self)
+      if job.respond_to?(:perform)
+        start!
+        STDOUT.reopen(@output_file) && STDERR.reopen(@output_file)
+        begin
+          job.perform
+          success!
+          Steve.log "[#{self.id}] Succeeded"
+        rescue Steve::Job::Delay => e
+          max_attempts = (Steve.max_job_retries || 5)
+          if self.retries >= max_attempts
+            fail!("#{e.message} after #{max_attempts} attempt(s)")
+            Steve.log "[#{self.id}] Failed after #{max_attempts} attempt(s): #{e.message}"
+          else
+            self.error = e.message
+            delay!
+            Steve.log "[#{self.id}] Delayed ('#{e.message}')"            
+          end
+        rescue Timeout::Error
+          fail!('Timed out')
+          Steve.log "[#{self.id}] Timed out: #{e.to_s}"
+        rescue => e
+          if e.is_a?(Steve::Job::Error)
+            fail!(e.message)
+          else
+            if defined?(Airbrake)
+              Airbrake.notify(e, :component => self.job.to_s, :action => self.id.to_s, :parameters => self.params)
+            end
+            fail!([e.to_s, e.backtrace].join("\n"))
+          end
+          Steve.log "[#{self.id}] Failed: #{e.to_s}"
+        end
+      else
+        fail! "#{self.id} did not respond to 'perform'"
+        Steve.log "[#{self.id}] Failed: does not respond to 'perform'"
+        return false
+      end
+    ensure
+      STDOUT.flush && STDERR.flush
+      self.output = File.read(@output_file)
+      self.save(:validate => false)
+      FileUtils.rm(@output_file) if File.exist?(@output_file)
+    end
+    
+    ## Get a lock on this job. Returns true if the lock was successful
+    ## otherwise it returns false.
+    def lock
+      rows = self.class.update_all({:worker => Steve.worker_name}, {:id => self.id, :worker => nil})
+      if rows == 1
+        self.worker = Steve.worker_name
+        return true
+      else
+        return false
+      end
+    end
+    
+    ## Mark this job as succeeded successfully.
+    def success!
+      self.status = 'completed'
+      finish!
+    end
+    
+    ## Mark this job as failed.
+    def fail!(message)
+      self.error = message
+      self.status = 'failed'
+      finish!
+    end
+    
+    ## Mark this job as finished
+    def finish!
+      self.finished_at = Time.now.utc
+    end
+    
+    ## Mark this job as started
+    def start!
+      self.error = nil
+      self.finished_at = nil
+      self.started_at = Time.now.utc
+      self.status = 'running'
+      self.save(:validate => false)
+    end
+    
+    ## Delay this job by the time specified
+    def delay!(delay_time = 30.seconds)
+      self.status = 'delayed'
+      self.run_at = Time.now.utc + delay_time
+      self.started_at = nil
+      self.worker = nil
+      self.retries += 1
+      self.save(:validate => false)
+    end
+    
+    ## Associate this job with the pased active record object
+    def associate_with(object)
+      self.associated_object = object
+      self.save(:validate => false)
+    end
+    
+    ## Remove old completed jobs from the database
+    def self.cleanup
+      self.delete_all(["status = 'completed' and run_at < ?", 5.days.ago])
+    end
+    
+  end
+end

data/lib/steve/worker.rb

@@ -0,0 +1,31 @@
+module Steve
+  class Worker
+    
+    def initialize(queue)
+      @queue = queue
+    end
+    
+    def start
+      Steve.log "*** Starting job worker #{Steve.worker_name} (queue: #{@queue})"
+      
+      trap("TERM") { Steve.log("*** Exiting..."); $exit = true }
+      trap("INT")  { Steve.log("*** Exiting..."); $exit = true }
+      
+      loop do
+        jobs = Steve::QueuedJob.execute_jobs(@queue)
+        count = jobs.size
+        
+        unless count == 0
+          Steve.log "*** #{count} jobs processed"
+        else
+          break if $exit
+          $0 = "sj: waiting for jobs on #{@queue}"
+          sleep(Steve.worker_sleep_time || 5)
+        end
+        
+        break if $exit
+      end
+      
+    end
+  end
+end

data/lib/steve.rb

@@ -0,0 +1,53 @@
+module Steve
+  
+  class << self
+    
+    ## The default queue for new jobs
+    attr_accessor :default_job_queue
+    
+    ## The default priority for new jobs
+    attr_accessor :default_job_priority    
+    
+    ## The name of the table where jobs are stored (default 'jobs')
+    attr_accessor :jobs_table_name
+    
+    ## The period of time to wait before looking for new jobs
+    attr_accessor :worker_sleep_time
+    
+    ## The logger object for all output from steve
+    attr_accessor :logger
+    
+    ## The maximum number of times to retry a job
+    attr_accessor :max_job_retries
+    
+    ## Whether or not jobs should be queued or run in the background
+    attr_accessor :run_jobs_in_foreground
+    
+    ## Proc to run after forking
+    attr_accessor :after_job_fork
+    
+    ## Return the worker name for this current process/host
+    def worker_name
+      "host:#{Socket.gethostname} pid:#{Process.pid}" rescue "pid:#{Process.pid}"
+    end
+    
+    ## Set/return the logger object
+    def logger
+      @logger ||= Logger.new(File.join(Rails.root, 'log', 'jobs.log'))
+    end
+    
+    ## Log a new message
+    def log(message)
+      message.gsub!(/(\[\d+\])/) { "\e[33m#{$1}\e[0m" }
+      logger.info "\e[37m#{Time.now.utc.to_s(:db)}\e[0m #{message}"
+    end
+    
+  end
+  
+end
+
+require 'steve/job'
+require 'steve/queued_job'
+require 'steve/worker'
+require 'steve/interface'
+

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: steve
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- aTech Media
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-09 00:00:00.000000000Z
+dependencies: []
+description: 
+email: support@atechmedia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/steve/interface/public/style.css
+- lib/steve/interface/views/completed.haml
+- lib/steve/interface/views/failed.haml
+- lib/steve/interface/views/index.haml
+- lib/steve/interface/views/object.haml
+- lib/steve/interface/views/view.haml
+- lib/steve/interface.rb
+- lib/steve/job.rb
+- lib/steve/queued_job.rb
+- lib/steve/worker.rb
+- lib/steve.rb
+homepage: http://atechmedia.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Deployment Recipes for Appli
+test_files: []