creeper 0.0.3 → 0.0.4

data/README.md

@@ -2,14 +2,18 @@
 
 Can be used as an in place drop in for stalker but it is multi threaded so you can easily do more without using more memory
 
+*Note*: Creeper requires Ruby 1.9
+
 Creeper - an improvement on Stalker
-==========================================
+===================================
 
 The big difference is how you "work" jobs
 
-all you need is a thread count arguement :)
+all you need is a runner (thread) count argument :)
 
-    Creeper.work(<jobs>, <thread_count>)
+```
+Creeper.work([<job>, ...], <runner_count>)
+```
 
 [Beanstalkd](http://kr.github.com/beanstalkd/) is a fast, lightweight queueing backend inspired by mmemcached.
 
@@ -18,62 +22,81 @@ Queueing jobs
 
 From anywhere in your app:
 
-    require 'creeper'
+```ruby
+require 'creeper'
 
-    Creeper.enqueue('email.send', :to => 'joe@example.com')
-    Creeper.enqueue('post.cleanup.all')
-    Creeper.enqueue('post.cleanup', :id => post.id)
+Creeper.enqueue('email.send', :to => 'joe@example.com')
+Creeper.enqueue('post.cleanup.all')
+Creeper.enqueue('post.cleanup', :id => post.id)
+```
 
 Working jobs
 ------------
 
 In a standalone file, typically jobs.rb or worker.rb:
 
-    require 'creeper'
-    include Creeper
+```ruby
+require 'creeper'
+include Creeper::Creep
+
+job 'email.send' do |args|
+  Pony.send(:to => args['to'], :subject => "Hello there")
+end
+
+job 'post.cleanup.all' do |args|
+  Post.all.each do |post|
+    enqueue('post.cleanup', :id => post.id)
+  end
+end
+
+job 'post.cleanup' do |args|
+  Post.find(args['id']).cleanup
+end
 
-    job 'email.send' do |args|
-      Pony.send(:to => args['to'], :subject => "Hello there")
-    end
+# All of these Creeper.work calls are equivalent:
 
-    job 'post.cleanup.all' do |args|
-      Post.all.each do |post|
-        enqueue('post.cleanup', :id => post.id)
-      end
-    end
+Creeper.work(:all, 3)                                                 # work all jobs, 3 threads
+Creeper.work(nil, 3)                                                  # same as previous line
+Creeper.work([ 'email.send', 'post.cleanup.all', 'post.cleanup' ], 3) # same as previous line
 
-    job 'post.cleanup' do |args|
-      Post.find(args['id']).cleanup
-    end
-    
-    Creeper.work(<jobs>, <thread_count>)
+# Here we work just one job:
+Creeper.work('email.send', 5) # work 'email.send', 5 threads
+```
 
 Running
 -------
 
 First, make sure you have Beanstalkd installed and running:
 
-    $ sudo brew install beanstalkd
-    $ beanstalkd
+```bash
+$ sudo brew install beanstalkd
+$ beanstalkd
+```
 
 Creeper:
 
-    $ sudo gem install creeper
+```bash
+$ sudo gem install creeper
+```
 
 Error Handling
 -------------
 
 If you include an `error` block in your jobs definition, that block will be invoked when a worker encounters an error. You might use this to report errors to an external monitoring service:
 
-    error do |e, job, args|
-      Exceptional.handle(e)
-    end
+```ruby
+error do |e, job, args|
+  Exceptional.handle(e)
+end
+```
 
 Before filter
 -------------
 
 If you wish to run a block of code prior to any job:
 
-    before do |job|
-      puts "About to work #{job}"
-    end
+```ruby
+before do |job|
+  puts "About to work #{job}"
+end
+```

data/bin/creeper

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib') unless $LOAD_PATH.include?(File.dirname(__FILE__) + '/../lib')
+
+require 'creeper/launcher'
+require 'optparse'
+
+options = { runner_count: 1 }
+
+op = OptionParser.new("", 24, '  ') do |opts|
+  cmd = File.basename($0)
+  opts.banner = "Usage: #{cmd} [ruby options] [#{cmd} options]"
+
+  opts.separator "Ruby options:"
+
+  opts.on("-d", "--debug", "set debugging flags (set $DEBUG to true)") do
+    $DEBUG = true
+  end
+
+  opts.separator "#{cmd} options:"
+
+  opts.on("-D", "--daemonize", "run daemonized in the background") do |d|
+    options[:daemonize] = !!d
+  end
+
+  opts.on("-j", "--job-file FILE", "Creeper-specific job file") do |f|
+    options[:job_file] = f
+  end
+
+  opts.on("-r", "--runner-count NUMBER", "Threads to run (default: #{options[:runner_count]})") do |r|
+    options[:runner_count] = (r.to_i rescue 1)
+  end
+
+  opts.separator "Common options:"
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts.to_s.gsub(/^.*DEPRECATED.*$/s, '')
+    exit
+  end
+
+  opts.on_tail("-v", "--version", "Show version") do
+    puts "#{cmd} v#{Creeper::VERSION}"
+    exit
+  end
+
+  opts.parse! ARGV
+
+end
+
+unless options[:job_file]
+  $stderr.puts "ERROR: job file required", ''
+  puts op.to_s.gsub(/^.*DEPRECATED.*$/s, '')
+  exit 1
+end
+
+if $DEBUG
+  require 'pp'
+  pp(options)
+end
+
+Creeper::Launcher.launch!(options) if options[:daemonize]
+Creeper.new(options).start.join

data/creeper.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |gem|
   gem.version       = Creeper::VERSION
 
   gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'rake'
   gem.add_development_dependency 'rspec'
   # gem.add_development_dependency 'stalker'
 

data/examples/jack.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env creeper -j
+#
+# There are 2 ways to run this file:
+# creeper -j jack.rb
+# OR
+# ./jack.rb
+#
+# If you remove the shebang, it can also be run with:
+# ruby jack.rb
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib') unless $LOAD_PATH.include?(File.dirname(__FILE__) + '/../lib')
+
+require 'creeper'
+
+include Creeper::Creep
+
+job('jack.work') do |*args|
+  Creeper.logger.info "[JACK.WORK] #{$0.inspect} #{args.inspect}"
+end
+
+Creeper.work(:all, 1) unless defined?(Creeper::Launcher)

data/lib/creeper.rb

@@ -12,14 +12,10 @@ require 'kgio'
 require 'logger'
 
 require 'creeper/version'
+require 'creeper/creep'
 require 'creeper/session'
 require 'creeper/worker'
 
-$stdout.sync = $stderr.sync = true
-$stdin.binmode
-$stdout.binmode
-$stderr.binmode
-
 module Creeper
 
   # These hashes map Threads to Workers
@@ -47,9 +43,10 @@ module Creeper
   end
 
   attr_accessor :logger, :error_logger
-  attr_accessor :jobs, :patience, :runner_count, :soft_quit, :timeout
+  attr_accessor :job_file, :jobs, :patience, :ready_pipe, :runner_count, :soft_quit, :timeout
 
   extend self
+  extend Creeper::Creep
 
   ## utilities ##
 
@@ -74,6 +71,12 @@ module Creeper
 
   ### config ###
 
+  def job_file=(job_file)
+    (@job_file = job_file).tap do
+      require File.expand_path(job_file) if job_file
+    end
+  end
+
   def patience
     @patience ||= 60
   end
@@ -105,6 +108,14 @@ module Creeper
 
   ###
 
+  def new(options = {})
+    tap do
+      options.each do |key, value|
+        send("#{key}=", value) if respond_to?("#{key}=")
+      end
+    end
+  end
+
   def work(jobs = nil, runner_count = 1)
     self.jobs, self.runner_count = jobs, runner_count
 
@@ -134,6 +145,10 @@ module Creeper
 
     reset_proc_name
     logger.info "creeper process ready"
+    if ready_pipe
+      ready_pipe.syswrite($$.to_s)
+      ready_pipe = ready_pipe.close rescue nil
+    end
     begin
       reap_all_runners
       case SIG_QUEUE.shift
@@ -228,7 +243,6 @@ module Creeper
         GRAVEYARD[thread] = worker
       else
         logger.debug "creeper [murder] => hard quit" if $DEBUG
-        worker.session.disconnect
         thread.kill
         thread.join
       end
@@ -268,6 +282,10 @@ module Creeper
   end
 
   def proc_name(tag)
+    if defined?(Navy) and defined?($officer)
+      Creeper::START_CTX.merge!(Navy::Admiral::START_CTX)
+      tag = "(#{$officer.captain.label}) officer[#{$officer.number}] #{tag}"
+    end
     $0 = ([
       File.basename(Creeper::START_CTX[0]),
       tag
@@ -276,31 +294,4 @@ module Creeper
 
   ##
 
-  public
-
-  def clear!
-    @default_session.disconnect if @default_session
-    @default_session = nil
-  end
-
-  def default_session
-    @default_session ||= Creeper::Session.new
-  end
-
-  def enqueue(job, args = {}, opts = {})
-    default_session.enqueue(job, args, opts)
-  end
-
-  def job(name, &block)
-    default_session.job(name, &block)
-  end
-
-  def before(&block)
-    default_session.before(&block)
-  end
-
-  def error(&block)
-    default_session.error(&block)
-  end
-
 end

data/lib/creeper/creep.rb

@@ -0,0 +1,31 @@
+module Creeper
+  module Creep
+
+    def clear!
+      Creeper.default_session.disconnect if Creeper.instance_variable_defined?(:@default_session) and Creeper.instance_variable_get(:@default_session)
+      Creeper.instance_variable_set(:@default_session, nil)
+    end
+
+    def default_session
+      return Creeper.instance_variable_get(:@default_session) if Creeper.instance_variable_defined?(:@default_session)
+      Creeper.instance_variable_set(:@default_session, Creeper::Session.new)
+    end
+
+    def enqueue(job, args = {}, opts = {})
+      default_session.enqueue(job, args, opts)
+    end
+
+    def job(name, &block)
+      default_session.job(name, &block)
+    end
+
+    def before(&block)
+      default_session.before(&block)
+    end
+
+    def error(&block)
+      default_session.error(&block)
+    end
+
+  end
+end

data/lib/creeper/launcher.rb

@@ -0,0 +1,44 @@
+$stdout.sync = $stderr.sync = true
+$stdin.binmode
+$stdout.binmode
+$stderr.binmode
+
+require 'creeper'
+
+module Creeper
+  module Launcher
+
+    extend self
+
+    def launch!(options)
+      $stdin.reopen("/dev/null")
+
+      # grandparent - reads pipe, exits when master is ready
+      #  \_ parent  - exits immediately ASAP
+      #      \_ creeper master - writes to pipe when ready
+
+      rd, wr = IO.pipe
+      grandparent = $$
+      if fork
+        wr.close # grandparent does not write
+      else
+        rd.close # creeper master does not read
+        Process.setsid
+        exit if fork # parent dies now
+      end
+
+      if grandparent == $$
+        # this will block until Creeper.join runs (or it dies)
+        creeper_pid = (rd.readpartial(16) rescue nil).to_i
+        unless creeper_pid > 1
+          warn "creeper failed to start, check stderr log for details"
+          exit!(1)
+        end
+        exit 0
+      else # creeper master process
+        options[:ready_pipe] = wr
+      end
+    end
+
+  end
+end

data/lib/creeper/version.rb

@@ -1,3 +1,3 @@
 module Creeper
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/lib/creeper/worker.rb

@@ -68,7 +68,9 @@ module Creeper
     def prepare
       raise NoJobsDefined if session.handlers.empty?
 
-      jobs = options[:jobs] || session.all_jobs
+      jobs   = session.all_jobs if options[:jobs] == :all
+      jobs ||= options[:jobs] ? [*options[:jobs]] : session.all_jobs
+      jobs   = session.all_jobs if jobs.empty?
 
       jobs.each do |job|
         raise(NoSuchJob, job) unless session.handlers.has_key?(job)
@@ -107,16 +109,17 @@ module Creeper
       end
 
       job.delete
-      log_job_end(name)
+      log_job_end(name, args)
     rescue Beanstalk::NotConnected => e
       failed_connection(e)
     rescue SystemExit
-      puts "FART"
+
       raise
     rescue => e
       log_error exception_message(e)
       job.bury rescue nil
-      log_job_end(name, 'failed') if @job_begun
+      args ||= []
+      log_job_end(name, args, 'failed') if @job_begun
       if session.error_handler
         if session.error_handler.arity == 1
           session.error_handler.call(e)
@@ -133,7 +136,7 @@ module Creeper
     end
 
     def stop
-      logger.info "worker dying: #{Thread.current.inspect}"
+      logger.info "worker dying: (current=#{Thread.current.inspect}#{@thread.inspect}"
       session.disconnect
       @thread.kill
     end
@@ -150,23 +153,28 @@ module Creeper
 
     def log_job_begin(name, args)
       @working = true
-      args_flat = unless args.empty?
-        '(' + args.inject([]) do |accum, (key,value)|
-          accum << "#{key}=#{value}"
-        end.join(' ') + ')'
-      else
-        ''
-      end
+      args_flat = flatten_args(args)
 
       log [ "Working", name, args_flat ].join(' ')
       @job_begun = Time.now
     end
 
-    def log_job_end(name, failed=false)
+    def log_job_end(name, args, failed=false)
       @working = false
       ellapsed = Time.now - @job_begun
       ms = (ellapsed.to_f * 1000).to_i
-      log "Finished #{name} in #{ms}ms #{failed ? ' (failed)' : ''}"
+      args_flat = flatten_args(args)
+      log [ "Finished #{name} in #{ms}ms #{failed ? ' (failed)' : ''}", args_flat ].join(' ')
+    end
+
+    def flatten_args(args)
+      unless args.empty?
+        '(' + args.inject([]) do |accum, (key,value)|
+          accum << "#{key}=#{value}"
+        end.join(' ') + ')'
+      else
+        ''
+      end
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: creeper
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-15 00:00:00.000000000 Z
+date: 2012-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -78,7 +94,8 @@ dependencies:
 description: Stalker with threads
 email:
 - lyondhill@gmail.com
-executables: []
+executables:
+- creeper
 extensions: []
 extra_rdoc_files: []
 files:
@@ -88,8 +105,12 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/creeper
 - creeper.gemspec
+- examples/jack.rb
 - lib/creeper.rb
+- lib/creeper/creep.rb
+- lib/creeper/launcher.rb
 - lib/creeper/session.rb
 - lib/creeper/version.rb
 - lib/creeper/worker.rb
@@ -109,15 +130,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -2251731633976259583
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -2251731633976259583
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.20
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: A better solution for io bound jobs, same as stalker in functionality but
@@ -127,4 +154,3 @@ test_files:
 - spec/lib/creeper/worker_spec.rb
 - spec/lib/creeper_spec.rb
 - spec/spec_helper.rb
-has_rdoc: