gemerald_beanstalk 0.0.1

data/lib/gemerald_beanstalk/connection.rb

@@ -0,0 +1,170 @@
+class GemeraldBeanstalk::Connection
+
+  BEGIN_REQUEST_STATES = [:ready, :multi_part_request_in_progress]
+
+  attr_reader :beanstalk, :mutex, :tube_used, :tubes_watched
+  attr_writer :producer, :waiting, :worker
+
+
+  def alive?
+    return @inbound_state != :closed && @oubound_state != :closed
+  end
+
+
+  def begin_multi_part_request(multi_part_request)
+    return false unless outbound_ready?
+    @multi_part_request = multi_part_request
+    @outbound_state = :multi_part_request_in_progress
+    return true
+  end
+
+
+  def begin_request
+    return false unless BEGIN_REQUEST_STATES.include?(@outbound_state)
+    @outbound_state = :request_in_progress
+    return true
+  end
+
+
+  def close_connection
+    @inbound_state = @outbound_state = :closed
+    @connection.close_connection unless @connection.nil?
+  end
+
+
+  def complete_request
+    return false unless request_in_progress?
+    @outbound_state = :ready
+    return true
+  end
+
+
+  def execute(raw_command)
+    command = response = nil
+    @mutex.synchronize do
+      return if waiting? || request_in_progress?
+      puts "#{Time.now.to_f}: #{raw_command}" if ENV['VERBOSE']
+      if multi_part_request_in_progress?
+        (command = @multi_part_request).body = raw_command
+      else
+        command = GemeraldBeanstalk::Command.new(raw_command, self)
+        if !command.valid?
+          response = command.error
+        elsif command.multi_part_request?
+          return begin_multi_part_request(command)
+        end
+      end
+      begin_request
+    end
+    puts "#{Time.now.to_f}: #{command.inspect}" if ENV['VERBOSE']
+    # Execute command unless parsing already yielded a response
+    response ||= beanstalk.execute(command)
+    transmit(response) unless response.nil?
+  end
+
+
+  def ignore(tube, force = false)
+    return nil unless @tubes_watched.length > 1 || force
+    @tubes_watched.delete(tube)
+    return @tubes_watched.length
+  end
+
+
+  def inbound_ready?
+    return @inbound_state == :ready
+  end
+
+
+  def initialize(beanstalk, connection = nil)
+    @beanstalk = beanstalk
+    @connection = connection
+    @inbound_state = :ready
+    @mutex = Mutex.new
+    @outbound_state = :ready
+    @tube_used = 'default'
+    @tubes_watched = Set.new(%w[default])
+  end
+
+
+  def multi_part_request_in_progress?
+    return @outbound_state == :multi_part_request_in_progress
+  end
+
+
+  def outbound_ready?
+    return @outbound_state == :ready
+  end
+
+
+  def producer?
+    return !!@producer
+  end
+
+
+  def request_in_progress?
+    return @outbound_state == :request_in_progress
+  end
+
+
+  def response_received
+    return false unless waiting? || timed_out?
+    @inbound_state = :ready
+    return true
+  end
+
+
+  def timed_out?
+    return @inbound_state == :timed_out
+  end
+
+
+  def transmit(message)
+    return if !alive? || @connection.nil?
+    puts "#{Time.now.to_f}: #{message}" if ENV['VERBOSE']
+    @connection.send_data(message)
+    complete_request
+    response_received
+    return self
+  end
+
+
+  def use(tube_name)
+    @tube_used = tube_name
+  end
+
+
+  def wait(timeout = nil)
+    return false unless inbound_ready?
+    @wait_timeout = timeout
+    @inbound_state = :waiting
+    return true
+  end
+
+
+  def wait_timed_out
+    return false unless @inbound_state == :waiting
+    @wait_timeout = nil
+    @inbound_state = :timed_out
+    return true
+  end
+
+
+  def waiting?
+    return false unless @inbound_state == :waiting
+    return true if @wait_timeout.nil? || @wait_timeout > Time.now.to_f
+    wait_timed_out
+    return false
+  end
+
+
+  def watch(tube)
+    @tubes_watched << tube
+    return @tubes_watched.length
+  end
+
+
+  def worker?
+    return !!@worker
+  end
+
+end

data/lib/gemerald_beanstalk/job.rb

@@ -0,0 +1,229 @@
+class GemeraldBeanstalk::Job
+
+  MAX_JOB_PRIORITY = 2**32
+
+  INACTIVE_STATES = [:buried, :delayed]
+  RESERVED_STATES = [:deadline_pending, :reserved]
+  UPDATE_STATES = [:deadline_pending, :delayed, :reserved]
+
+  attr_reader :beanstalk, :reserved_at, :reserved_by, :timeout_at
+  attr_accessor :priority, :tube_name, :delay, :ready_at, :body,
+    :bytes, :created_at, :ttr, :id, :buried_at
+
+
+  def <(other_job)
+    return (self <=> other_job) == -1
+  end
+
+
+  def <=>(other_job)
+    raise 'Cannot compare job with nil' if other_job.nil?
+    current_state = state
+    raise 'Cannot compare jobs with different states' if current_state != other_job.state
+
+    case current_state
+    when :ready
+      return -1 if self.priority < other_job.priority ||
+        self.priority == other_job.priority && self.created_at < other_job.created_at
+    when :delayed
+      return -1 if self.ready_at < other_job.ready_at
+    when :buried
+      return -1 if self.buried_at < other_job.buried_at
+    else
+      raise "Cannot compare job with state of #{current_state}"
+    end
+    return 1
+  end
+
+
+  def buried?
+    return state == :buried
+  end
+
+
+  def bury(connection, priority, *args)
+    return false unless reserved_by_connection?(connection)
+
+    reset_reserve_state
+    @state = :buried
+    @stats_hash[:'buries'] += 1
+    self.priority = priority.to_i
+    self.buried_at = Time.now.to_f
+    self.ready_at = nil
+    return true
+  end
+
+
+  # Must look at @state to avoid infinite recursion
+  def deadline_approaching(*args)
+    return false unless @state == :reserved
+    @state = :deadline_pending
+    return true
+  end
+
+
+  def deadline_pending?
+    return state == :deadline_pending
+  end
+
+
+  def delayed?
+    return state == :delayed
+  end
+
+
+  def delete(connection, *args)
+    return false if RESERVED_STATES.include?(state) && !reserved_by_connection?(connection)
+    @state = :deleted
+    return true
+  end
+
+
+  def initialize(beanstalk, id, tube_name, priority, delay, ttr, bytes, body)
+    priority, delay, ttr = priority.to_i, delay.to_i, ttr.to_i
+    @beanstalk = beanstalk
+    @stats_hash = Hash.new(0)
+    self.id = id
+    self.tube_name = tube_name
+    self.priority = priority % MAX_JOB_PRIORITY
+    self.delay = delay
+    self.ttr = ttr == 0 ? 1 : ttr
+    self.bytes = bytes
+    self.body = body
+    self.created_at = Time.now.to_f
+    self.ready_at = self.created_at + delay
+
+    @state = delay > 0 ? :delayed : :ready
+  end
+
+
+  def kick(*args)
+    return false unless INACTIVE_STATES.include?(state)
+
+    @state = :ready
+    @stats_hash[:'kicks'] += 1
+    self.ready_at = Time.now.to_f
+    self.buried_at = nil
+    return true
+  end
+
+
+  def ready?
+    return state == :ready
+  end
+
+
+  def release(connection, priority, delay, increment_stats = true, *args)
+    return false unless reserved_by_connection?(connection)
+
+    delay = delay.to_i
+    reset_reserve_state
+    @state = delay > 0 ? :delayed : :ready
+    @stats_hash[:'releases'] += 1 if increment_stats
+    self.priority = priority.to_i
+    self.delay = delay
+    self.ready_at = Time.now.to_f + delay
+    return true
+  end
+
+
+  def reserve(connection, *args)
+    return false unless ready?
+
+    @state = :reserved
+    @stats_hash[:'reserves'] += 1
+    @reserved_by = connection
+    @reserved_at = Time.now.to_f
+    @timeout_at = @reserved_at + self.ttr
+    return true
+  end
+
+
+  def reserved_by_connection?(connection)
+    return RESERVED_STATES.include?(state) && self.reserved_by == connection ? true : false
+  end
+
+
+  def reset_reserve_state
+    @timeout_at = nil
+    @reserved_at = nil
+    @reserved_by = nil
+  end
+
+
+  def state
+    return @state unless UPDATE_STATES.include?(@state)
+
+    now = Time.now.to_f
+    if @state == :delayed && self.ready_at <= now
+      @state = :ready
+    elsif RESERVED_STATES.include?(@state)
+      # Rescue from timeout being reset by other thread
+      if (now > self.timeout_at rescue false)
+        timed_out
+      elsif (@state == :reserved && now + 1 > self.timeout_at rescue false)
+        deadline_approaching
+      end
+    end
+
+    return @state
+  end
+
+
+  def stats
+    now = Time.now.to_f
+    current_state = state
+    return {
+      'id' => self.id,
+      'tube' => self.tube_name,
+      'state' => current_state == :deadline_pending ? 'reserved' : current_state.to_s,
+      'pri' => self.priority,
+      'age' => (now - self.created_at).to_i,
+      'delay' => self.delay.to_i,
+      'ttr' => self.ttr,
+      'time-left' => time_left(now),
+      'file' => 0,
+      'reserves' => @stats_hash[:'reserves'],
+      'timeouts' => @stats_hash[:'timeouts'],
+      'releases' => @stats_hash[:'releases'],
+      'buries' => @stats_hash[:'buries'],
+      'kicks' => @stats_hash[:'kicks'],
+    }
+  end
+
+
+  def time_left(current_time = Time.now.to_f)
+    if self.timeout_at
+      time_left = self.timeout_at - current_time
+    elsif self.ready_at
+      time_left = self.ready_at - current_time
+    end
+    return time_left.to_i
+  end
+
+
+  # Must reference @state to avoid infinite recursion
+  def timed_out(*args)
+    return false unless RESERVED_STATES.include?(@state)
+    @state = :ready
+    @stats_hash[:'timeouts'] += 1
+    connection = self.reserved_by
+    reset_reserve_state
+    self.beanstalk.register_job_timeout(connection, self)
+    return true
+  end
+
+
+  def timed_out?
+    return state == :timed_out
+  end
+
+
+  def touch(connection)
+    return false unless reserved_by_connection?(connection)
+    @state = :reserved
+    @timeout_at = Time.now.to_f + self.ttr
+    return true
+  end
+
+end

data/lib/gemerald_beanstalk/jobs.rb

@@ -0,0 +1,39 @@
+class GemeraldBeanstalk::Jobs < ThreadSafe::Array
+  attr_reader :total_jobs
+
+  def counts_by_state
+    job_stats = {
+      'current-jobs-urgent' => 0,
+      'current-jobs-ready' => 0,
+      'current-jobs-reserved' => 0,
+      'current-jobs-delayed' => 0,
+      'current-jobs-buried' => 0,
+    }
+    self.compact.each do |job|
+      state = job.state
+
+      job_stats["current-jobs-#{state}"] += 1
+      job_stats['current-jobs-urgent'] += 1 if state == :ready && job.priority < 1024
+    end
+    return job_stats
+  end
+
+
+  def enqueue(job)
+    @total_jobs += 1
+    push(job)
+    return self
+  end
+
+
+  def initialize(*)
+    @total_jobs = 0
+    super
+  end
+
+
+  def next_id
+    return @total_jobs + 1
+  end
+
+end

data/lib/gemerald_beanstalk/server.rb

@@ -0,0 +1,54 @@
+require 'eventmachine'
+
+module GemeraldBeanstalk::Server
+
+  def self.start(bind_address = nil, port = nil)
+    bind_address ||= '0.0.0.0'
+    port ||= 11300
+    full_address = "#{bind_address}:#{port}"
+    beanstalk = GemeraldBeanstalk::Beanstalk.new(full_address)
+    thread = Thread.new do
+      EventMachine.run do
+        EventMachine.start_server(bind_address, port, GemeraldBeanstalk::Server, beanstalk)
+        EventMachine.add_periodic_timer(0.01, beanstalk.method(:update_state))
+      end
+    end
+    $PROGRAM_NAME = "gemerald_beanstalk:#{full_address}"
+    return [thread, beanstalk]
+  end
+
+
+  def beanstalk
+    return @beanstalk
+  end
+
+
+  def initialize(beanstalk)
+    @beanstalk = beanstalk
+    @partial_message = ''
+    super
+  end
+
+
+  def post_init
+    @connection = beanstalk.connect(self)
+  end
+
+
+  def receive_data(data)
+    if data[-2, 2] == "\r\n"
+      message = @partial_message + data
+      @partial_message = ''
+      EventMachine.defer(proc { @connection.execute(message) })
+    else
+      @partial_message += data
+    end
+  end
+
+
+  def unbind
+    beanstalk.disconnect(@connection)
+    @connection.close_connection
+  end
+
+end