creeper 1.0.9 → 2.0.0

data/lib/creeper/web.rb

@@ -0,0 +1,248 @@
+require 'sinatra/base'
+require 'slim'
+require 'sprockets'
+require 'creeper/paginator'
+
+module Creeper
+  class SprocketsMiddleware
+    def initialize(app, options={})
+      @app = app
+      @root = options[:root]
+      path   =  options[:path] || 'assets'
+      @matcher = /^\/#{path}\/*/
+      @environment = ::Sprockets::Environment.new(@root)
+      @environment.append_path 'assets/javascripts'
+      @environment.append_path 'assets/javascripts/vendor'
+      @environment.append_path 'assets/stylesheets'
+      @environment.append_path 'assets/stylesheets/vendor'
+      @environment.append_path 'assets/images'
+    end
+
+    def call(env)
+      # Solve the problem of people requesting /creeper when they need to request /creeper/ so
+      # that relative links in templates resolve correctly.
+      return [301, { 'Location' => "#{env['SCRIPT_NAME']}/", 'Content-Type' => 'text/html' }, ['redirecting']] if env['SCRIPT_NAME'] == env['REQUEST_PATH']
+
+      return @app.call(env) unless @matcher =~ env["PATH_INFO"]
+      env['PATH_INFO'].sub!(@matcher,'')
+      @environment.call(env)
+    end
+  end
+
+  class Web < Sinatra::Base
+    include Creeper::Paginator
+
+    dir = File.expand_path(File.dirname(__FILE__) + "/../../web")
+    set :views,  "#{dir}/views"
+    set :root, "#{dir}/public"
+    set :slim, :pretty => true
+    use SprocketsMiddleware, :root => dir
+
+    helpers do
+
+      def reset_worker_list
+        Creeper.redis do |conn|
+          workers = conn.smembers('workers')
+          workers.each do |name|
+            conn.srem('workers', name)
+          end
+        end
+      end
+
+      def workers
+        @workers ||= begin
+          Creeper.redis do |conn|
+            conn.smembers('workers').map do |w|
+              msg = conn.get("worker:#{w}")
+              msg ? [w, Creeper.load_json(msg)] : nil
+            end.compact.sort { |x| x[1] ? -1 : 1 }
+          end
+        end
+      end
+
+      def processed
+        Creeper.redis { |conn| conn.get('stat:processed') } || 0
+      end
+
+      def failed
+        Creeper.redis { |conn| conn.get('stat:failed') } || 0
+      end
+
+      def zcard(name)
+        Creeper.redis { |conn| conn.zcard(name) }
+      end
+
+      def queues
+        @queues ||= Creeper.redis do |conn|
+          conn.smembers('queues').map do |q|
+            [q, conn.llen("queue:#{q}") || 0]
+          end.sort { |x,y| x[1] <=> y[1] }
+        end
+      end
+
+      def backlog
+        # queues.map {|name, size| size }.inject(0) {|memo, val| memo + val }
+        beanstalk_stats['current-jobs-ready']
+      end
+
+      def buried
+        beanstalk_stats['current-jobs-buried']
+      end
+
+      def scheduled
+        beanstalk_stats['current-jobs-delayed']
+      end
+
+      def retries_with_score(score)
+        Creeper.redis do |conn|
+          results = conn.zrangebyscore('retry', score, score)
+          results.map { |msg| Creeper.load_json(msg) }
+        end
+      end
+
+      def beanstalk_stats
+        @beanstalk_stats ||= Creeper.beanstalk do |conn|
+          conn.stats
+        end
+      end
+
+      def beanstalk_location
+        Creeper.beanstalk { |conn| conn.instance_variable_get(:@addrs) }.join(', ')
+      end
+
+      def location
+        Creeper.redis { |conn| conn.client.location }
+      end
+
+      def root_path
+        "#{env['SCRIPT_NAME']}/"
+      end
+
+      def current_status
+        return 'idle' if workers.size == 0
+        return 'active'
+      end
+
+      def relative_time(time)
+        %{<time datetime="#{time.getutc.iso8601}">#{time}</time>}
+      end
+
+      def display_args(args, count=100)
+        args.map { |arg| a = arg.inspect; a.size > count ? "#{a[0..count]}..." : a }.join(", ")
+      end
+    end
+
+    get "/" do
+      slim :index
+    end
+
+    get "/poll" do
+      slim :poll, layout: false
+    end
+
+    get "/queues" do
+      @queues = queues
+      slim :queues
+    end
+
+    get "/queues/:name" do
+      halt 404 unless params[:name]
+      @count = (params[:count] || 25).to_i
+      @name = params[:name]
+      (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
+      @messages = @messages.map {|msg| Creeper.load_json(msg) }
+      slim :queue
+    end
+
+    post "/reset" do
+      reset_worker_list
+      redirect root_path
+    end
+
+    post "/queues/:name" do
+      Creeper.redis do |conn|
+        conn.del("queue:#{params[:name]}")
+        conn.srem("queues", params[:name])
+      end
+      redirect "#{root_path}queues"
+    end
+
+    get "/retries/:score" do
+      halt 404 unless params[:score]
+      @score = params[:score].to_f
+      @retries = retries_with_score(@score)
+      redirect "#{root_path}retries" if @retries.empty?
+      slim :retry
+    end
+
+    get '/retries' do
+      @count = (params[:count] || 25).to_i
+      (@current_page, @total_size, @retries) = page("retry", params[:page], @count)
+      @retries = @retries.map {|msg, score| [Creeper.load_json(msg), score] }
+      slim :retries
+    end
+
+    get '/scheduled' do
+      @count = (params[:count] || 25).to_i
+      (@current_page, @total_size, @scheduled) = page("schedule", params[:page], @count)
+      @scheduled = @scheduled.map {|msg, score| [Creeper.load_json(msg), score] }
+      slim :scheduled
+    end
+
+    post '/scheduled' do
+      halt 404 unless params[:score]
+      halt 404 unless params['delete']
+      params[:score].each do |score|
+        s = score.to_f
+        process_score('schedule', s, :delete)
+      end
+      redirect "#{root_path}scheduled"
+    end
+
+    post '/retries' do
+      halt 404 unless params[:score]
+      params[:score].each do |score|
+        s = score.to_f
+        if params['retry']
+          process_score('retry', s, :retry)
+        elsif params['delete']
+          process_score('retry', s, :delete)
+        end
+      end
+      redirect "#{root_path}retries"
+    end
+
+    post "/retries/:score" do
+      halt 404 unless params[:score]
+      score = params[:score].to_f
+      if params['retry']
+        process_score('retry', score, :retry)
+      elsif params['delete']
+        process_score('retry', score, :delete)
+      end
+      redirect "#{root_path}retries"
+    end
+
+    def process_score(set, score, operation)
+      case operation
+      when :retry
+        raise "Not Implemented"
+        Creeper.redis do |conn|
+          results = conn.zrangebyscore(set, score, score)
+          conn.zremrangebyscore(set, score, score)
+          results.map do |message|
+            msg = Creeper.load_json(message)
+            msg['retry_count'] = msg['retry_count'] - 1
+            conn.rpush("queue:#{msg['queue']}", Creeper.dump_json(msg))
+          end
+        end
+      when :delete
+        Creeper.redis do |conn|
+          conn.zremrangebyscore(set, score, score)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/creeper/worker.rb

@@ -1,331 +1,80 @@
-require 'creeper'
-
-require 'celluloid'
-require 'creeper/celluloid_ext'
-# require 'em-jack'
+require 'creeper/client'
+require 'creeper/core_ext'
 
 module Creeper
 
-  at_exit { shutdown_workers }
-
-  class Worker
-
-    def self.work(jobs = nil, size = nil)
-
-      size ||= Creeper.pool_size
-
-      options = {
-        size: size,
-        args: [jobs]
-      }
-
-      Creeper.worker_pool = Creeper::Worker.pool(options)
-
-      begin
-        trap(:INT)  { Creeper.shutdown = true }
-        trap(:TERM) { Creeper.shutdown = true }
-        trap(:QUIT) { Creeper.shutdown = true }
-        Creeper.worker_pool.start
-      end until Creeper.shutdown?
-
-      exit
-    end
-
-    # def self.em_work(jobs = nil, size = 2)
-
-    #   options = {
-    #     size: size,
-    #     args: [jobs]
-    #   }
-
-    #   tubes = jobs_for(jobs)
-
-    #   Creeper.worker_pool = Creeper::Worker.pool(options)
-
-    #   sleep 1
-
-    #   EM.run do
-    #     trap(:INT)  { Creeper.shutdown = true; EM.stop }
-    #     trap(:TERM) { Creeper.shutdown = true; EM.stop }
-    #     trap(:QUIT) { Creeper.shutdown = true; EM.stop }
-
-    #     jack = EMJack::Connection.new(Creeper.beanstalk_url)
-
-    #     reserve_loop = ->(timeout) do
-    #       r = jack.reserve(timeout)
-    #       r.callback do |job|
-    #         Creeper.worker_pool.work! job
-    #         EM.next_tick { reserve_loop.call(timeout) }
-    #       end
-    #       r.errback do
-    #         EM.next_tick { reserve_loop.call(timeout) }
-    #       end
-    #     end
-
-    #     watch_tubes = ->(list) do
-    #       if list.empty?
-    #         reserve_loop.call(Creeper.reserve_timeout)
-    #       else
-    #         w = jack.watch(list.shift)
-    #         w.callback do
-    #           watch_tubes.call(list)
-    #         end
-    #       end
-    #     end
-
-    #     watch_tubes.call(tubes)
-
-    #   end
-
-    # end
-
-    def self.jobs_for(jobs = nil)
-      case jobs
-      when :all, nil
-        Creeper.all_jobs
-      else
-        Array(jobs)
-      end
-    end
-
-    include Celluloid
-
-    attr_accessor :number
-    attr_reader   :jobs
-
-    def initialize(jobs = nil)
-      @jobs = self.class.jobs_for(jobs)
-
-      Creeper.register_worker(self)
-      OutLogger.info "#{prefix} Working #{self.jobs.size} jobs: [ #{self.jobs.join(' ')} ]"
-    end
-
-    def dump(job, name = nil, data = nil)
-      "#{name.inspect rescue nil} { data: #{(data.inspect rescue nil)}, job: #{(job.inspect rescue nil)} }"
-    end
-
-    def prefix
-      "[#{number_format % number} - #{'%x' % Thread.current.object_id}]"
-    end
-
-    def number_format
-      "%#{Creeper.pool_size.to_s.length}d"
-    end
-
-    def time_in_milliseconds
-      ((stopped_at - started_at).to_f * 1000).to_i
-    end
-
-    def started_at
-      Thread.current[:creeper_started_at]
-    end
-
-    def started_at=(started_at)
-      Thread.current[:creeper_started_at] = started_at
-    end
-
-    def stopped_at
-      Thread.current[:creeper_stopped_at]
-    end
-
-    def stopped_at=(stopped_at)
-      Thread.current[:creeper_stopped_at] = stopped_at
-    end
-
-    ## beanstalk ##
-
-    def beanstalk
-      Creeper.beanstalk
-    end
-
-    def ignore(tube)
-      beanstalk.ignore(tube)
-    rescue Beanstalk::NotConnected => e
-      disconnected(self, :ignore, tube) || raise
-    end
-
-    def list_tubes_watched(cached = false)
-      beanstalk.list_tubes_watched
-    rescue Beanstalk::NotConnected => e
-      disconnected(self, :list_tubes_watched, cached) || raise
-    end
-
-    def reserve(timeout = nil)
-      beanstalk.reserve(timeout)
-    rescue Beanstalk::NotConnected => e
-      disconnected(self, :reserve, timeout) || raise
-    end
-
-    def watch(tube)
-      beanstalk.watch(tube)
-    rescue Beanstalk::NotConnected => e
-      disconnected(self, :watch, tube) || raise
-    end
-
-    ##
-
-    ## work ##
-
-    def start(short_circuit = false)
-      return false if short_circuit
-      exit         if stopped?
-      return true  if working?
-      prepare      if not prepared?
-
-      begin
-        job = reserve Creeper.reserve_timeout
-      rescue Beanstalk::TimedOut
-        OutLogger.debug "#{prefix} Back to the unemployment line" if $DEBUG
-        return false
+  ##
+  # Include this module in your worker class and you can easily create
+  # asynchronous jobs:
+  #
+  # class HardWorker
+  #   include Creeper::Worker
+  #
+  #   def perform(*args)
+  #     # do some work
+  #   end
+  # end
+  #
+  # Then in your Rails app, you can do this:
+  #
+  #   HardWorker.perform_async(1, 2, 3)
+  #
+  # Note that perform_async is a class method, perform is an instance method.
+  module Worker
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.class_attribute :creeper_options_hash
+    end
+
+    def logger
+      Creeper.logger
+    end
+
+    module ClassMethods
+
+      include Creeper::Legacy::WorkerMethods
+
+      def perform_async(*args)
+        client_push('class' => self, 'args' => args)
       end
 
-      exit if stopped?
-
-      Thread.current[:creeper_working] = true
-
-      OutLogger.debug "#{prefix} Got #{job.inspect}" if $DEBUG
-
-      work! job # asynchronously go to work
-    rescue SystemExit => e
-      job.release rescue nil
-      Creeper.unregister_worker(self)
-    rescue => e
-      job.release rescue nil
-      Creeper.unregister_worker(self, "start loop error")
-      raise
-    end
-
-    def finalize
-      Creeper.finalizers.each do |finalizer|
-        begin
-          case finalizer.arity
-          when 1
-            finalizer.call(self)
-          else
-            finalizer.call
-          end
-        rescue => e
-          OutLogger.crash "#{prefix} finalizer error", e
-        end
+      def perform_in(interval, *args)
+        int = interval.to_f
+        ts = (int < 1_000_000_000 ? Time.now.to_f + int : int)
+        client_push('class' => self, 'args' => args, 'at' => ts)
       end
-
-    ensure
-      Creeper.disconnect
-    end
-
-    def stop
-      Thread.current[:creeper_stopped] = true
-    end
-
-    def work(job)
-
-      exit if stopped?
-
-      name, data = JSON.parse(job.body)
-
-      Creeper.start_work(self, data, name, job)
-
-      begin
-        Creeper.before_handlers_for(name).each do |handler|
-          process(handler, data, name, job)
-        end
-
-        Creeper.handler_for(name).tap do |handler|
-          process(handler, data, name, job)
-        end
-
-        Creeper.after_handlers_for(name).each do |handler|
-          process(handler, data, name, job)
-        end
+      alias_method :perform_at, :perform_in
+
+      ##
+      # Allows customization for this type of Worker.
+      # Legal options:
+      #
+      #   :queue - use a named queue for this Worker, default 'default'
+      #   :retry - enable the RetryJobs middleware for this Worker, default *true*
+      #   :timeout - timeout the perform method after N seconds, default *nil*
+      #   :backtrace - whether to save any error backtrace in the retry payload to display in web UI,
+      #      can be true, false or an integer number of lines to save, default *false*
+      def creeper_options(opts={})
+        self.creeper_options_hash = get_creeper_options.merge(stringify_keys(opts || {}))
       end
 
-      job.delete
+      DEFAULT_OPTIONS = { 'retry' => true, 'queue' => 'default' }
 
-      Creeper.stop_work(self, data, name, job)
-
-      # start! unless stopped? or EM.reactor_running?
-      start! unless stopped? # continue processing, even when end of links is reached
-
-    rescue Beanstalk::NotConnected => e
-      disconnected(self, :work, job) || begin
-        job.release rescue nil
-        Creeper.unregister_worker(self)
-        raise
+      def get_creeper_options # :nodoc:
+        self.creeper_options_hash ||= DEFAULT_OPTIONS
       end
-    rescue SystemExit => e
-      job.release rescue nil
-      Creeper.unregister_worker(self)
-    rescue => e
-
-      job.bury rescue nil
-
-      Creeper.error_work(self, data, name, job)
 
-      begin
-        Creeper.error_handlers_for(name).each do |handler|
-          process(handler, data, name, job)
+      def stringify_keys(hash) # :nodoc:
+        hash.keys.each do |key|
+          hash[key.to_s] = hash.delete(key)
         end
+        hash
       end
 
-      Creeper.unregister_worker(self, "work loop error, burying #{dump(job, name, data)}")
-
-      raise
-    ensure
-      Thread.current[:creeper_started_at] = nil
-      Thread.current[:creeper_stopped_at] = nil
-      Thread.current[:creeper_working] = false
-    end
-
-    def process(handler, data, name, job)
-      case handler.arity
-      when 3
-        handler.call(data, name, job)
-      when 2
-        handler.call(data, name)
-      when 1
-        handler.call(data)
-      else
-        handler.call
-      end
-    end
-
-    ##
-
-    ## flags ##
-
-    def prepared?
-      Thread.current[:creeper_prepared] == true
-    end
-
-    def stopped?
-      Thread.current[:creeper_stopped] == true || Creeper.shutdown?
-    end
-
-    def working?
-      Thread.current[:creeper_working] == true
-    end
-
-    ##
-
-    protected
-
-    def disconnected(target, method, *args, &block)
-      Creeper.send(:disconnected, target, method, *args, &block)
-    end
-
-    def prepare
-      jobs.each do |job|
-        watch(job)
-      end
-
-      list_tubes_watched.each do |server, tubes|
-        tubes.each do |tube|
-          ignore(tube) unless jobs.include?(tube)
-        end
+      def client_push(*args) # :nodoc:
+        Creeper::Client.push(*args)
       end
 
-      Thread.current[:creeper_prepared] = true
     end
-
   end
-end
+end