kongnomal 0.1.3

data/bin/kongnomal

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "optparse"
+require "ostruct"
+require "socket"
+require "kongnomal"
+
+options = OpenStruct.new
+
+OptionParser.new do |opts|
+  opts.banner = "USAGE: kongnomal [options] job_event_handler_file"
+
+  opts.on("-c", "--connect-to [beanstalk_address]", "address of beanstalk process to connect to, in form of ip_address:port_number") do |config|
+    options.ip_address, options.port_number = config.split(":")
+    puts("beanstalk_address #{config} should be in form of ip_address:port_number") || exit unless options.ip_address and options.port_number
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+options.job_event_handler = ARGV[0]
+
+require File.expand_path(options.job_event_handler) if options.job_event_handler
+unless options.ip_address and options.port_number
+  port_number, ip = begin
+    (dummy_server = TCPServer.new("127.0.0.1", 0)).addr[1..2]
+  ensure
+    dummy_server.close
+  end
+  puts "starting beanstalk server with new beanstalkd process at #{ip}:#{port_number}"
+  Kongnomal::BeanstalkServer.new.start(ip, port_number)
+else
+  puts "starting beanstalk server with existing beanstalkd process at #{options.ip_address}:#{options.port_number}"
+  Kongnomal::BeanstalkServer.new.connect_to(options.ip_address, options.port_number)
+end

data/example/job_event_handler.rb

@@ -0,0 +1,21 @@
+Kongnomal::Job.bind do |job|
+  job.when_touched do |event|
+    #do something when worker has given feedback
+  end
+  
+  job.when_put do |event|
+    #do something when job is put into queue
+  end
+  
+  job.when_reserved do |event|
+    #do something when job is reserved by a worker
+  end
+  
+  job.when_deleted do |event|
+    #do something when job is deleted from tube
+  end
+  
+  job.when_released do |event|
+    #do something when job is released
+  end
+end

data/example/worker.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "kongnomal"
+
+beanstalk_address = ARGV[0]
+tube = ARGV[1]
+
+jobs = Kongnomal::JobQueue.new(beanstalk_address, tube)
+until jobs.peek_ready.nil? do
+  begin
+    job = jobs.reserve(1)
+    job.report_back({:state => "start"})
+    job.report_back({:state => "in_progress"})
+    job.report_back({:state => "success"})
+    job.delete
+  rescue Exception => e
+  end
+end

data/example/workers.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "kongnomal"
+
+ip_address = ARGV[0]
+port_number = ARGV[1]
+
+jobs = Kongnomal::JobQueue.new("#{ip_address}:#{port_number}", "jobs")
+
+300.times do |i|
+  @pid = Process.spawn("./worker.rb '#{ip_address}:#{port_number}' 'jobs'")
+  Process.detach(@pid)
+end
+
+100000.times do |i|
+  jobs.put("job#{i}")
+end

data/lib/kongnomal.rb

@@ -0,0 +1,7 @@
+require "forwardable"
+require "beanstalk-client"
+require "json"
+require "yaml"
+require File.join(File.dirname(__FILE__), "kongnomal", "beanstalk_server")
+require File.join(File.dirname(__FILE__), "kongnomal", "job_queue")
+require File.join(File.dirname(__FILE__), "kongnomal", "job")

data/lib/kongnomal/beanstalk_server.rb

@@ -0,0 +1,74 @@
+module Kongnomal
+  class BeanstalkServer
+    def initialize(&blk)
+      @on_startup = block_given? ? blk : Proc.new { puts "beanstalk server is up and running..." }
+      at_exit {stop_beanstalkd}
+      self
+    end
+  
+    def start(ip, port_number)
+      start_beanstalkd(ip, port_number)
+      connect_to(ip, port_number)
+    end
+  
+    def connect_to(ip, port_number)
+      job_event_queue = job_event_queue("#{ip}:#{port_number}", Kongnomal::JobQueue::JOB_EVENT_TUBE)
+      @on_startup.call
+    
+      loop do
+        job = begin
+          job_event_queue.reserve
+        rescue Exception => e
+          puts "Failed to retrieve job event from #{ip}:#{port_number}[#{Kongnomal::JobQueue::JOB_EVENT_TUBE}]"
+          raise e
+        end
+
+        job_evented = JSON.parse(job.body)
+        begin
+          Kongnomal::Job.send(job_evented["state"]).call(job_evented)
+        rescue Exception => e
+          job.release(nil, 60)
+          puts "Failed to process job event: #{job_evented.inspect}, retrying 60 sec later..."
+          puts e.message
+          puts e.backtrace
+          next
+        end
+      
+        begin
+          job.delete
+        rescue Exception => e
+          puts "Failed to delete job event: #{job_evented.inspect} from #{ip}:#{port_number}[#{Kongnomal::JobQueue::JOB_EVENT_TUBE}], ignoring for now to be reprocessed after natural beanstalk job timeout"
+          puts e.message
+          puts e.backtrace
+        end
+      end
+    end
+  
+    private
+    def start_beanstalkd(ip, port_number)
+      stop_beanstalkd
+      @pid = Process.spawn("beanstalkd -l #{ip} -p #{port_number}", :unsetenv_others => true, :close_others => true)
+      Process.detach(@pid)
+      puts "beanstalkd process(#{@pid}) has started at #{ip}:#{port_number}"
+    end
+  
+    def stop_beanstalkd
+      puts "beanstalkd process(#{@pid}) has been stopped" if @pid and system("kill #{@pid}")
+    end
+  
+    def job_event_queue(beanstalk_server_address, tube)
+      job_event_queue = 3.times do |i|
+        begin
+          puts "connecting to #{tube} tube at #{beanstalk_server_address}"
+          break Beanstalk::Connection.new(beanstalk_server_address, tube)
+        rescue Exception => e
+          raise e if i + 1 == 3
+          puts "reconnecting in 1 second..."
+          sleep(1)
+        end
+      end
+      puts "listening for #{tube} tube at #{beanstalk_server_address}"
+      job_event_queue
+    end
+  end
+end

data/lib/kongnomal/job.rb

@@ -0,0 +1,72 @@
+module Kongnomal
+  class Job
+    PUT = "put".freeze
+    RESERVED = "reserved".freeze
+    DELETED = "deleted".freeze
+    TOUCHED = "touched".freeze
+    RELEASED = "released".freeze
+    
+    def self.when_put(&blk)
+      @put_handler = blk
+    end
+    
+    def self.when_reserved(&blk)
+      @reserved_handler = blk
+    end
+    
+    def self.when_touched(&blk)
+      @touched_handler = blk
+    end
+    
+    def self.when_deleted(&blk)
+      @deleted_handler = blk
+    end
+    
+    def self.when_released(&blk)
+      @released_handler = blk
+    end
+    
+    class << self
+      [PUT, RESERVED, TOUCHED, DELETED, RELEASED].each do |event|
+        define_method(event.to_sym) do
+          Proc.new do |job|
+            event_handler = instance_variable_get("@#{event}_handler")
+            event_handler.call(job) if event_handler
+          end
+        end
+      end
+      
+      def bind(&blk)
+        yield self
+      end
+    end
+    
+    extend Forwardable
+    def_delegators :@beanstalk_job, :id, :body, :ybody, :put_back, :bury, :stats, :timeouts, :time_left, :age, :state, :delay, :pri, :ttr, :server, :decay, :to_s, :inspect
+  
+    def initialize(event_tube, beanstalk_job, tube)
+      @event_tube = event_tube
+      @beanstalk_job = beanstalk_job
+      @tube = tube
+    end
+    
+    def touch(data)
+      as_msg = data.respond_to?(:to_hash) ? data.to_hash : data.to_s
+      touched = @beanstalk_job.touch
+      @event_tube.put({"state" => Kongnomal::Job::TOUCHED, "id" => id, "body" => body, "tube" => @tube, "msg" => as_msg}.to_json) if touched == :ok
+      touched
+    end
+    
+    def release(*args, &blk)
+      released = @beanstalk_job.release(*args, &blk)
+      @event_tube.put({"state" => Kongnomal::Job::RELEASED, "id" => id, "body" => body, "tube" => @tube}.to_json) unless released.nil?
+      released
+    end
+    
+    def delete(*args, &blk)
+      deleted = @beanstalk_job.delete(*args, &blk)
+      @event_tube.put({"state" => Kongnomal::Job::DELETED, "id" => id, "body" => body, "tube" => @tube}.to_json) unless deleted.nil?
+      deleted
+    end
+  end
+end

data/lib/kongnomal/job_queue.rb

@@ -0,0 +1,39 @@
+module Kongnomal
+  class JobQueue
+    JOB_EVENT_TUBE = "job.events"
+    
+    extend Forwardable
+    def_delegators :@job_tube, :peek_job, :peek_ready, :peek_delayed, :peek_buried, :bury, :kick, :stats, :job_stats, :stats_tube, :list_tubes, :list_tube_used, :list_tubes_watched
+  
+    def initialize(beanstalk_server_address, tube = "default")
+      @tube = tube
+      @job_tube = Beanstalk::Connection.new(beanstalk_server_address, tube)
+      @event_tube = Beanstalk::Connection.new(beanstalk_server_address, JOB_EVENT_TUBE)
+    end
+    
+    def put(body, pri=65536, delay=0, ttr=120)
+      job_id = @job_tube.put(body, pri, delay, ttr)
+      @event_tube.put({"state" => Kongnomal::Job::PUT, "tube" => @tube, "id" => job_id, "body" => body}.to_json)
+      job_id
+    end
+    
+    def yput(obj, pri=65536, delay=0, ttr=120)
+      job_id = @job_tube.yput(obj, pri, delay, ttr)
+      @event_tube.put({"state" => Kongnomal::Job::PUT, "tube" => @tube, "id" => job_id, "body" => YAML.dump(obj)}.to_json)
+      job_id
+    end
+  
+    def reserve(timeout=nil)
+      beanstalk_job = @job_tube.reserve(timeout)
+      @event_tube.put({"state" => Kongnomal::Job::RESERVED, "tube" => @tube, "id" => beanstalk_job.id, "body" => beanstalk_job.body}.to_json)
+      Kongnomal::Job.new(@event_tube, beanstalk_job, @tube)
+    end
+    
+    def delete(id)
+      beanstalk_job = @job_tube.peek_job(id)
+      deleted = @job_tube.delete(id)
+      @event_tube.put({"state" => Kongnomal::Job::DELETED, "tube" => @tube, "id" => beanstalk_job.id, "body" => beanstalk_job.body}.to_json) if deleted == :ok
+      deleted
+    end
+  end
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification 
+name: kongnomal
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 3
+  version: 0.1.3
+platform: ruby
+authors: 
+- Jae Lee
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-05 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: beanstalk-client
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 1
+        - 0
+        version: 1.1.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: agent
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: Register your custom job event handler that gets notified upon job event triggered from job.report_back or job.delete_then_report_back
+email: jlee@yetitrails.com
+executables: 
+- kongnomal
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/kongnomal
+- lib/kongnomal.rb
+- lib/kongnomal/beanstalk_server.rb
+- lib/kongnomal/job.rb
+- lib/kongnomal/job_queue.rb
+- example/job_event_handler.rb
+- example/worker.rb
+- example/workers.rb
+has_rdoc: true
+homepage: https://github.com/forward/kongnomal
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: beanstalk server wrapper that allows event handler registration on job event queue
+test_files: []
+