creeper 0.0.5 → 1.0.0

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in creeper.gemspec
 gemspec
+
+gem 'celluloid', git: 'git://github.com/celluloid/celluloid.git'

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012 lyon
+Copyright (c) 2012 Andrew Bennett
 
 MIT License
 

data/README.md

@@ -1,102 +1,29 @@
 # Creeper
 
-Can be used as an in place drop in for stalker but it is multi threaded so you can easily do more without using more memory
+TODO: Write a gem description
 
-*Note*: Creeper requires Ruby 1.9
+## Installation
 
-Creeper - an improvement on Stalker
-===================================
+Add this line to your application's Gemfile:
 
-The big difference is how you "work" jobs
+    gem 'creeper'
 
-all you need is a runner (thread) count argument :)
+And then execute:
 
-```
-Creeper.work([<job>, ...], <runner_count>)
-```
+    $ bundle
 
-[Beanstalkd](http://kr.github.com/beanstalkd/) is a fast, lightweight queueing backend inspired by mmemcached.
+Or install it yourself as:
 
-Queueing jobs
--------------
+    $ gem install creeper
 
-From anywhere in your app:
+## Usage
 
-```ruby
-require 'creeper'
+TODO: Write usage instructions here
 
-Creeper.enqueue('email.send', :to => 'joe@example.com')
-Creeper.enqueue('post.cleanup.all')
-Creeper.enqueue('post.cleanup', :id => post.id)
-```
+## Contributing
 
-Working jobs
-------------
-
-In a standalone file, typically jobs.rb or worker.rb:
-
-```ruby
-require 'creeper'
-include Creeper::Creep
-
-job 'email.send' do |args|
-  Pony.send(:to => args['to'], :subject => "Hello there")
-end
-
-job 'post.cleanup.all' do |args|
-  Post.all.each do |post|
-    enqueue('post.cleanup', :id => post.id)
-  end
-end
-
-job 'post.cleanup' do |args|
-  Post.find(args['id']).cleanup
-end
-
-# All of these Creeper.work calls are equivalent:
-
-Creeper.work(:all, 3)                                                 # work all jobs, 3 threads
-Creeper.work(nil, 3)                                                  # same as previous line
-Creeper.work([ 'email.send', 'post.cleanup.all', 'post.cleanup' ], 3) # same as previous line
-
-# Here we work just one job:
-Creeper.work('email.send', 5) # work 'email.send', 5 threads
-```
-
-Running
--------
-
-First, make sure you have Beanstalkd installed and running:
-
-```bash
-$ sudo brew install beanstalkd
-$ beanstalkd
-```
-
-Creeper:
-
-```bash
-$ sudo gem install creeper
-```
-
-Error Handling
--------------
-
-If you include an `error` block in your jobs definition, that block will be invoked when a worker encounters an error. You might use this to report errors to an external monitoring service:
-
-```ruby
-error do |e, job, args|
-  Exceptional.handle(e)
-end
-```
-
-Before filter
--------------
-
-If you wish to run a block of code prior to any job:
-
-```ruby
-before do |job|
-  puts "About to work #{job}"
-end
-```
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -1,6 +1,2 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
-Bundler::GemHelper.install_tasks
-
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new('spec')

data/creeper.gemspec

@@ -2,11 +2,11 @@
 require File.expand_path('../lib/creeper/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["lyon"]
-  gem.email         = ["lyondhill@gmail.com"]
-  gem.description   = %q{Stalker with threads}
-  gem.summary       = %q{A better solution for io bound jobs, same as stalker in functionality but more threadie.}
-  gem.homepage      = "https://github.com/lyondhill/creeper"
+  gem.authors       = ["Lyon Hill", "Andrew Bennett"]
+  gem.email         = ["lyondhill@gmail.com", "potatosaladx@gmail.com"]
+  gem.description   = %q{Creeper is an evented version of Stalker}
+  gem.summary       = %q{A better solution for io bound jobs, same as stalker in functionality but more evented}
+  gem.homepage      = "https://github.com/potatosalad/creeper"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -18,10 +18,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'rspec'
-  # gem.add_development_dependency 'stalker'
 
   gem.add_dependency 'beanstalk-client'
+  gem.add_dependency 'celluloid'
+  gem.add_dependency 'em-jack'
   gem.add_dependency 'kgio'
-
 end
-

data/lib/creeper.rb

@@ -1,297 +1,405 @@
 require 'beanstalk-client'
-
 require 'json'
-require 'uri'
-require 'timeout'
-
-require 'fcntl'
-require 'etc'
-require 'stringio'
-require 'kgio'
-
 require 'logger'
+require 'thread'
+require 'timeout'
+require 'uri'
 
 require 'creeper/version'
-require 'creeper/creep'
-require 'creeper/session'
-require 'creeper/worker'
 
 module Creeper
 
-  # These hashes map Threads to Workers
-  RUNNERS = {}
-  GRAVEYARD = {}
+  class BadURL < RuntimeError; end
+
+  HANDLERS = {
+    named:        {},
+    before_each:  [],
+    before_named: {},
+    after_each:   [],
+    after_named:  {},
+    error_each:   [],
+    error_named:  {},
+    finalizers:   []
+  }
 
-  SELF_PIPE = []
+  WORKERS = {}
 
-  # signal queue used for self-piping
-  SIG_QUEUE = []
+  ## default configuration ##
 
-  # list of signals we care about and trap in Creeper
-  QUEUE_SIGS = [ :WINCH, :QUIT, :INT, :TERM, :TTIN, :TTOU ]
+  @beanstalk_url   = ENV['BEANSTALK_URL'] || 'beanstalk://127.0.0.1/'
+  @logger          = ::Logger.new($stderr)
+  @patience_soft   = 60
+  @patience_hard   = 30
+  @pool_size       = 2
+  @retry_count     = 3
+  @reserve_timeout = 1
 
-  START_CTX = {
-    :argv => ARGV.map { |arg| arg.dup },
-    0 => $0.dup,
-  }
-  START_CTX[:cwd] = begin
-    a = File.stat(pwd = ENV['PWD'])
-    b = File.stat(Dir.pwd)
-    a.ino == b.ino && a.dev == b.dev ? pwd : Dir.pwd
-  rescue
-    Dir.pwd
-  end
+  @lock = Mutex.new
 
-  attr_accessor :logger, :error_logger
-  attr_accessor :job_file, :jobs, :patience, :ready_pipe, :runner_count, :soft_quit, :timeout
+  ##
 
-  extend self
-  extend Creeper::Creep
+  class << self
 
-  ## utilities ##
+    ## configuration ##
 
-  def logger
-    @logger ||= Logger.new($stdout)
-  end
+    attr_reader   :lock
+    attr_accessor :beanstalk_url, :logger, :patience_soft, :patience_hard, :pool_size, :reserve_timeout, :retry_count
 
-  def log_exception(prefix, exc, logger = error_logger)
-    message = exc.message
-    message = message.dump if /[[:cntrl:]]/ =~ message
-    logger.error "#{prefix}: #{message} (#{exc.class})"
-    exc.backtrace.each { |line| logger.error(line) }
-  end
+    def worker_pool
+      lock.synchronize do
+        @worker_pool
+      end
+    end
 
-  def error_logger
-    @error_logger ||= Logger.new($stderr)
-  end
+    def worker_pool=(worker_pool)
+      lock.synchronize do
+        @worker_pool = worker_pool
+      end
+    end
 
-  ##
+    def shutdown?
+      lock.synchronize do
+        !!@shutdown
+      end
+    end
 
-  ## main process ##
+    def shutdown=(shutdown)
+      lock.synchronize do
+        @shutdown = shutdown
+      end
+    end
+
+    ##
 
-  ### config ###
+    ## connection ##
 
-  def job_file=(job_file)
-    (@job_file = job_file).tap do
-      require File.expand_path(job_file) if job_file
+    def beanstalk
+      Thread.current[:beanstalk_pool_connection] ||= connect
     end
-  end
 
-  def patience
-    @patience ||= 60
-  end
+    def beanstalk_addresses
+      uris = beanstalk_url.split(/[\s,]+/)
+      uris.map do |uri|
+        beanstalk_host_and_port(uri)
+      end
+    end
 
-  def runner_count
-    @runner_count ||= 1
-  end
+    def connect(addresses = nil)
+      Beanstalk::Pool.new(addresses || beanstalk_addresses)
+    end
 
-  def runner_count=(value)
-    (@runner_count = value).tap do
-      reset_proc_name
+    def disconnect
+      Thread.current[:beanstalk_pool_connection].close rescue nil
+      Thread.current[:beanstalk_pool_connection] = nil
     end
-  end
 
-  def soft_quit
-    @soft_quit ||= false
-  end
-  alias :soft_quit? :soft_quit
+    ##
+
+    ## daemon ##
+
+    def work(jobs = nil, size = 2)
+      require 'creeper/worker'
 
-  def soft_quit=(soft_quit)
-    (@soft_quit = soft_quit).tap do
-      awaken_creeper if soft_quit?
+      Creeper::Worker.work(jobs, size)
     end
-  end
 
-  def timeout
-    @timeout ||= 30
-  end
+    ##
 
-  ###
+    ## handlers ##
 
-  def new(options = {})
-    tap do
-      options.each do |key, value|
-        send("#{key}=", value) if respond_to?("#{key}=")
+    def all_jobs
+      lock.synchronize do
+        HANDLERS[:named].keys
       end
     end
-  end
 
-  def work(jobs = nil, runner_count = 1)
-    self.jobs, self.runner_count = jobs, runner_count
+    def job(name, &block)
+      lock.synchronize do
+        HANDLERS[:named][name] = block
+        HANDLERS[:before_named][name] ||= []
+        HANDLERS[:after_named][name]  ||= []
+        HANDLERS[:error_named][name]  ||= []
+        HANDLERS[:named][name]
+      end
+    end
 
-    default_session.beanstalk # check if we can connect to beanstalk
+    def drop(name)
+      lock.synchronize do
+        HANDLERS[:named].delete(name)
+        HANDLERS[:before_named].delete(name)
+        HANDLERS[:after_named].delete(name)
+        HANDLERS[:error_named].delete(name)
+        true
+      end
+    end
 
-    start.join
-  end
+    def handler_for(name)
+      lock.synchronize do
+        HANDLERS[:named][name]
+      end
+    end
 
-  def start
-    init_self_pipe!
-    QUEUE_SIGS.each do |sig|
-      trap(sig) do
-        logger.debug "creeper received #{sig}" if $DEBUG
-        SIG_QUEUE << sig
-        awaken_creeper
+    def before(name = nil, &block)
+      if name and name != :each
+        lock.synchronize do
+          HANDLERS[:before_named][name] << block
+        end
+      else
+        lock.synchronize do
+          HANDLERS[:before_each] << block
+        end
       end
     end
 
-    logger.info "creeper starting"
+    def before_handlers_for(name)
+      lock.synchronize do
+        HANDLERS[:before_each] + HANDLERS[:before_named][name]
+      end
+    end
 
-    self
-  end
+    def after(name = nil, &block)
+      if name and name != :each
+        lock.synchronize do
+          HANDLERS[:after_named][name] << block
+        end
+      else
+        lock.synchronize do
+          HANDLERS[:after_each] << block
+        end
+      end
+    end
 
-  def join
-    respawn = true
-    last_check = Time.now
-
-    reset_proc_name
-    logger.info "creeper process ready"
-    if ready_pipe
-      ready_pipe.syswrite($$.to_s)
-      ready_pipe = ready_pipe.close rescue nil
-    end
-    begin
-      reap_all_runners
-      case SIG_QUEUE.shift
-      when nil
-        # break if soft_quit?
-        # avoid murdering runners after our master process (or the
-        # machine) comes out of suspend/hibernation
-        if (last_check + timeout) >= (last_check = Time.now)
-          sleep_time = timeout - 1
-        else
-          sleep_time = timeout/2.0 + 1
-          logger.debug("creeper waiting #{sleep_time}s after suspend/hibernation") if $DEBUG
+    def after_handlers_for(name)
+      lock.synchronize do
+        HANDLERS[:after_each] + HANDLERS[:after_named][name]
+      end
+    end
+
+    def error(name = nil, &block)
+      if name and name != :each
+        lock.synchronize do
+          HANDLERS[:error_named][name] << block
+        end
+      else
+        lock.synchronize do
+          HANDLERS[:error_each] << block
         end
-        maintain_runner_count if respawn
-        logger.debug("creeper sleeping for #{sleep_time}s") if $DEBUG
-        creeper_sleep(sleep_time)
-      when :QUIT # graceful shutdown
-        break
-      when :TERM, :INT # immediate shutdown
-        stop(false)
-        break
-      when :WINCH
-        self.runner_count = 0
-        logger.debug "WINCH: setting runner_count to #{runner_count}" if $DEBUG
-      when :TTIN
-        self.runner_count += 1
-        logger.debug "TTIN: setting runner_count to #{runner_count}" if $DEBUG
-      when :TTOU
-        self.runner_count -= 1 if runner_count > 0
-        logger.debug "TTOU: setting runner_count to #{runner_count}" if $DEBUG
       end
-    rescue => e
-      Creeper.log_exception("creeper loop error", e)
-    end while true
-    stop # gracefully shutdown all captains on our way out
-    logger.info "creeper complete"
-  end
+    end
 
-  def stop(graceful = true)
-    limit = Time.now + patience
-    kill_all_runners
-    until (RUNNERS.empty? && GRAVEYARD.empty?) || (n = Time.now) > limit
-      reap_graveyard(graceful)
-      sleep(0.1)
+    def error_handlers_for(name)
+      lock.synchronize do
+        HANDLERS[:error_each] + HANDLERS[:error_named][name]
+      end
     end
-    if n and n > limit
-      logger.debug "creeper patience exceeded by #{n - limit} seconds (limit #{patience} seconds)" if $DEBUG
+
+    def finalizer(&block)
+      lock.synchronize do
+        HANDLERS[:finalizers] << block
+      end
+    end
+
+    def finalizers
+      lock.synchronize do
+        HANDLERS[:finalizers]
+      end
     end
-    reap_graveyard(false)
-    logger.debug graceful ? "creeper gracefully stopped" : "creeper hard stopped" if $DEBUG
-  end
 
-  private
+    ##
 
-  # wait for a signal hander to wake us up and then consume the pipe
-  def creeper_sleep(sec)
-    IO.select([ SELF_PIPE[0] ], nil, nil, sec) or return
-    SELF_PIPE[0].kgio_tryread(11)
-  end
+    ## queue ##
 
-  def awaken_creeper
-    SELF_PIPE[1].kgio_trywrite('.') # wakeup creeper process from select
-  end
+    def enqueue(job, data = {}, options = {})
+      # Logger.debug "#{Thread.current[:actor].inspect} Enqueueing #{job.inspect}, #{data.inspect}"#\n#{Celluloid::Actor.all.pretty_inspect}"
+      Logger.debug "[#{Thread.current[:actor] ? Thread.current[:actor].subject.number : nil}] Enqueueing #{job.inspect}, #{data.inspect}" if $DEBUG
+      enqueue!(job, data, options)
+    rescue Beanstalk::NotConnected => e
+      disconnected(self, :enqueue, job, data, options)
+    end
 
-  def init_self_pipe!
-    SELF_PIPE.each { |io| io.close rescue nil }
-    SELF_PIPE.replace(Kgio::Pipe.new)
-    SELF_PIPE.each { |io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
-  end
+    def enqueue!(job, data = {}, options = {})
+      priority    = options[:priority] || options[:pri] || 65536
+      delay       = [ 0, options[:delay].to_i ].max
+      time_to_run = options[:time_to_run] || options[:ttr] || 120
 
-  def kill_all_runners
-    RUNNERS.each do |thread, worker|
-      GRAVEYARD[thread] = RUNNERS.delete(thread)
+      beanstalk.use job
+      beanstalk.put JSON.dump([ job, data ]), priority, delay, time_to_run
     end
-  end
 
-  def maintain_runner_count
-    current_runner_count = RUNNERS.size - runner_count
+    ##
 
-    spawn_missing_runners if current_runner_count < 0
-    murder_extra_runners  if current_runner_count > 0
-    reap_all_runners
-    reap_graveyard
-  end
+    ## workers ##
 
-  def murder_extra_runners
-    until RUNNERS.size == runner_count
-      thread, worker = RUNNERS.shift
-      if worker.working?
-        logger.debug "creeper [murder] => soft quit" if $DEBUG
-        worker.soft_quit = true
-        GRAVEYARD[thread] = worker
-      else
-        logger.debug "creeper [murder] => hard quit" if $DEBUG
-        thread.kill
-        thread.join
+    def error_work(worker, data, name, job)
+      (worker.stopped_at = Time.now).tap do |stopped_at|
+        Logger.error "#{worker.prefix} Error in #{worker.time_in_milliseconds}ms #{worker.dump(job, name, data)}"
       end
     end
-  end
 
-  def reap_all_runners
-    RUNNERS.each do |thread, worker|
-      GRAVEYARD[thread] = worker if not thread.alive?
+    def register_worker(worker)
+      lock.synchronize do
+        number = ((0..(WORKERS.keys.max || 0)+1).to_a - WORKERS.keys).first
+        WORKERS[number] = worker.tap do
+          worker.number = number
+        end
+      end
     end
-  end
 
-  def reap_graveyard(graceful = true)
-    GRAVEYARD.each do |thread, worker|
-      if graceful and worker.working?
-        logger.debug "creeper [graveyard] => soft quit" if $DEBUG
-        worker.soft_quit = true
-      else
-        logger.debug "creeper [graveyard] => hard quit" if $DEBUG
-        thread.kill rescue nil
-        thread.join
-        GRAVEYARD.delete(thread)
+    def shutdown_workers
+      begin
+        soft_shutdown_workers(Creeper.patience_soft)
+      rescue Timeout::Error
+        begin
+          hard_shutdown_workers(Creeper.patience_hard)
+        rescue Timeout::Error
+          kill_shutdown_workers
+        end
+      end
+    end
+
+    def start_work(worker, data, name, job)
+      (worker.started_at = Time.now).tap do |started_at|
+        Logger.info "#{worker.prefix} Working #{Thread.list.count} #{worker.dump(job, name, data)}"
       end
     end
-  end
 
-  def spawn_missing_runners
-    until RUNNERS.size == runner_count
-      worker = Creeper::Worker.new(jobs: jobs)
-      thread = worker.start
-      RUNNERS[thread] = worker
+    def stop_work(worker, data, name, job)
+      (worker.stopped_at = Time.now).tap do |stopped_at|
+        Logger.info "#{worker.prefix} Finished in #{worker.time_in_milliseconds}ms #{worker.dump(job, name, data)}"
+      end
     end
-  end
 
-  def reset_proc_name
-    proc_name "creeper(#{$$}) [#{runner_count}]"
-  end
+    def unregister_worker(worker, reason = nil)
+      reason ||= 'Stopping'
+      Logger.info "#{worker.prefix} #{reason}"
+      lock.synchronize do
+        WORKERS.delete(worker.number)
+      end
+    end
+
+    ##
+
+    protected
 
-  def proc_name(tag)
-    if defined?(Navy) and defined?($officer)
-      Creeper::START_CTX.merge!(Navy::Admiral::START_CTX)
-      tag = "(#{$officer.captain.label}) officer[#{$officer.number}] #{tag}"
+    def beanstalk_host_and_port(uri_string)
+      uri = URI.parse(uri_string)
+      raise(BadURL, uri_string) if uri.scheme != 'beanstalk'
+      "#{uri.host}:#{uri.port || 11300}"
     end
-    $0 = ([
-      File.basename(Creeper::START_CTX[0]),
-      tag
-    ]).concat(Creeper::START_CTX[:argv]).join(' ')
-  end
 
-  ##
+    def disconnected(target, method, *args, &block)
+      Thread.current[:beanstalk_connection_retries] ||= 0
+
+      if Thread.current[:beanstalk_connection_retries] >= retry_count
+        Logger.error "Unable to connect to beanstalk after #{Thread.current[:beanstalk_connection_retries]} attempts"
+        Thread.current[:beanstalk_connection_retries] = 0
+        return false
+      end
+
+      disconnect
+
+      Thread.current[:beanstalk_connection_retries] += 1
+
+      sleep Thread.current[:beanstalk_connection_retries] * 2
+
+      target.send(method, *args, &block)
+    end
+
+    def soft_shutdown_workers(timeout)
+      Timeout.timeout(timeout) do
+        actors = Celluloid::Actor.all
+        Logger.info "Gracefully stopping #{actors.size} actors..." if actors.size > 0
+
+        # Attempt to shut down the supervision tree, if available
+        Celluloid::Supervisor.root.terminate if Celluloid::Supervisor.root
+
+        # Actors cannot self-terminate, you must do it for them
+        starts = working_actors.map do |actor|
+          begin
+            if actor.alive?
+              actor.stop! # fire and forget for those already working
+              actor.future(:start, true) # ensures that the mailbox is cleared out
+            end
+          rescue Celluloid::DeadActorError, Celluloid::MailboxError
+          end
+        end.compact
+
+        starts.each do |start|
+          begin
+            start.value
+          rescue Celluloid::DeadActorError, Celluloid::MailboxError
+          end
+        end
+
+        Logger.info "Graceful stop completed cleanly"
+      end
+    end
+
+    def hard_shutdown_workers(timeout)
+      Timeout.timeout(timeout) do
+        actors = Celluloid::Actor.all
+        Logger.info "Terminating #{actors.size} actors..." if actors.size > 0
+
+        # Attempt to shut down the supervision tree, if available
+        Celluloid::Supervisor.root.terminate if Celluloid::Supervisor.root
+
+        pool_managers.each do |pool_manager|
+          begin
+            pool_manager.terminate
+          rescue Celluloid::DeadActorError, Celluloid::MailboxError
+          end
+        end
+
+        # Actors cannot self-terminate, you must do it for them
+        working_actors.each do |actor|
+          begin
+            actor.terminate
+          rescue Celluloid::DeadActorError, Celluloid::MailboxError
+          end
+        end
+
+        Logger.info "Termination completed cleanly"
+      end
+    end
+
+    def kill_shutdown_workers
+      actors = Celluloid::Actor.all
+      Logger.info "Killing #{actors.size} actors..." if actors.size > 0
+
+      # Attempt to shut down the supervision tree, if available
+      Celluloid::Supervisor.root.kill if Celluloid::Supervisor.root
+
+      # Actors cannot self-terminate, you must do it for them
+      Celluloid::Actor.all.each do |actor|
+        begin
+          actor.kill
+          actor.join
+        rescue Celluloid::DeadActorError, Celluloid::MailboxError
+        end
+      end
+
+      Logger.info "Killing completed cleanly"
+    end
+
+    def pool_managers
+      Celluloid::Actor.all.tap do |actors|
+        actors.keep_if do |actor|
+          actor.is_a?(Celluloid::PoolManager) rescue false
+        end
+      end
+    end
+
+    def working_actors
+      Celluloid::Actor.all.tap do |actors|
+        actors.delete_if do |actor|
+          actor.is_a?(Celluloid::PoolManager) rescue false
+        end
+      end
+    end
+
+  end
 
 end
+
+# require 'creeper/creep'
+require 'creeper/logger'