hydroponic_bean 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f4b6d8a2f04c35eb3f1633af6644cd013a72ed8
-  data.tar.gz: 30490d78a6e123d415091462c65d8eee5f663d67
+  metadata.gz: 9a0b2b9b746d52cb5ed0499c0ac2f8dc45213231
+  data.tar.gz: 4f23cda0c022146a7c5b99cb043363cc6a59ae8d
 SHA512:
-  metadata.gz: 374d7a1c86c90bb0c4e11300910ef30cae7ce3b9b1867ea61df180a086049b9813b1628ffe25645cf72f2ecf7fe4a695bb0d16e4fcc6c1508bd38987f7523fde
-  data.tar.gz: 65fbc11688e4fd1770890e6542c9d9f8d7c5b52ba8488b918518750ca6355fa57be1d2e2771d340dd3dfa174651e74bdb76554839555e0893dcb946b135855c9
+  metadata.gz: 0e0d194be39a4af19c053f946e9eda3740e3e1dbf17791608920abbd2bf5e2eb178804436eea5ec3c7be9e2177afef5991a52e81256133b943eab42120f16aa1
+  data.tar.gz: de72c9b293986ced6d10fc67952e8d1b2af2ac50896bbea77079931cc8bd4f96e1e994e82842f9cf008fbe21890bc0a9b43ff2bcd1ed12dda8ec01386f240432

data/.ruby-version

@@ -0,0 +1 @@
+2.3.0

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 1.2.0
+
+- Added all the worker commands
+- Added HydroponicBean::Connection.closed?
+- Now works as a beanstalk backend for Backburner!
+
 # 1.1.0
 
 - Added support for a lot more commands, see lib/hydroponic_bean/commands/

data/Gemfile

@@ -1,4 +1,7 @@
 source 'https://rubygems.org'
 
+gem 'simplecov'
+gem 'timecop'
+
 # Specify your gem's dependencies in hydroponic_bean.gemspec
 gemspec

data/lib/hydroponic_bean/commands.rb

@@ -9,5 +9,17 @@ module HydroponicBean
     include HydroponicBean::Commands::Worker
     include HydroponicBean::Commands::Tube
     include HydroponicBean::Commands::Other
+
+    # Find a job by id and yield it
+    #
+    # Outputs Protocol::NOT_FOUND if the block returns false
+    # or the job is not found
+    def for_job(id)
+      job = HydroponicBean.find_job(id)
+      if !job || !yield(job)
+        output(Protocol::NOT_FOUND)
+        return false
+      end
+    end
   end
 end

data/lib/hydroponic_bean/commands/other.rb

@@ -2,12 +2,13 @@ module HydroponicBean
   module Commands
     module Other
       def peek(stream, id = nil)
-        id = id.to_i
-        job = (id == 0) ? nil : HydroponicBean.jobs[id - 1]
-        peek_output(job)
+        for_job(id) do |job|
+          peek_output(job)
+        end
       end
 
       def peek_ready(stream)
+        HydroponicBean.update_time!
         peek_output current_tube.ready_jobs.first
       end
 
@@ -16,46 +17,23 @@ module HydroponicBean
       end
 
       def peek_delayed(stream)
+        HydroponicBean.update_time!
         peek_output current_tube.delayed_jobs.first
       end
 
       def stats_job(stream, id)
-        id = id.to_i
-        job = (id == 0) ? nil : HydroponicBean.jobs[id - 1]
-        if !job
-          output(Protocol::NOT_FOUND)
-          return false
+        for_job(id) do |job|
+          stats = job.serialize_stats.to_yaml
+          output("OK #{stats.length}\r\n")
+          output("#{stats}\r\n")
         end
-
-        stats = job.serialize_stats.to_yaml
-        output("OK #{stats.length}\r\n")
-        output("#{stats}\r\n")
       end
 
       def kick_job(stream, id)
-        id = id.to_i
-        job = (id == 0) ? nil : HydroponicBean.jobs[id - 1]
-        if !job || !job.kick
-          output(Protocol::NOT_FOUND)
-          return false
-        end
-        job.kick
-        output("KICKED\r\n")
-      end
-
-      def watch(stream, tube_name)
-        watched_tube_names << tube_name
-        watched_tube_names.uniq!
-        output("WATCHING #{watched_tube_names.count}\r\n")
-      end
-
-      def ignore(stream, tube_name)
-        watched_tube_names.delete(tube_name)
-        if watched_tube_names.empty?
-          watched_tube_names << tube_name
-          output("NOT_IGNORED\r\n")
-        else
-          output("WATCHING #{watched_tube_names.count}\r\n")
+        for_job(id) do |job|
+          if job.kick
+            output("KICKED\r\n")
+          end
         end
       end
 

data/lib/hydroponic_bean/commands/producer.rb

@@ -7,6 +7,9 @@ module HydroponicBean
       end
 
       def put(stream, pri, delay, ttr, bytes)
+        # Mark this connection as a producer
+        producer!
+
         bytes = bytes.to_i
         data = stream.read(bytes)
 
@@ -20,8 +23,8 @@ module HydroponicBean
           return false
         end
 
-        id = create_job(pri, delay, ttr, data)
-        output("INSERTED #{id}\r\n")
+        job = create_job(pri, delay, ttr, data)
+        output("INSERTED #{job.id}\r\n")
       end
     end
   end

data/lib/hydroponic_bean/commands/tube.rb

@@ -1,17 +1,6 @@
 module HydroponicBean
   module Commands
     module Tube
-      def delete(stream, id)
-        job = HydroponicBean.jobs[id.to_i - 1]
-        if job
-          job.delete
-          output("DELETED\r\n")
-        else
-          output(Protocol::NOT_FOUND)
-          return false
-        end
-      end
-
       def list_tubes(stream)
         tubes = HydroponicBean.tubes.keys.to_yaml
         output("OK #{tubes.length}\r\n")
@@ -48,6 +37,7 @@ module HydroponicBean
       end
 
       def kick(stream, bound)
+        HydroponicBean.update_time!
         bound = bound.to_i
         tube = current_tube
         output("KICKED #{tube.kick(bound)}\r\n")

data/lib/hydroponic_bean/commands/worker.rb

@@ -1,6 +1,82 @@
 module HydroponicBean
   module Commands
     module Worker
+      def reserve(stream)
+        reserve_with_timeout(stream, -1)
+      end
+
+      def reserve_with_timeout(stream, seconds)
+        # Mark this connection as a worker
+        worker!
+
+        if deadline_soon?
+          output("DEADLINE_SOON\r\n")
+          return true
+        end
+
+        seconds = seconds.to_i
+
+        if job = wait_for_job(seconds)
+          output("RESERVED #{job.id} #{job.data.length}\r\n")
+          output("#{job.data}\r\n")
+        else
+          output("TIMED_OUT\r\n")
+        end
+      end
+
+      def release(stream, id, pri, delay)
+        # We don't have a BURIED response here
+        for_job(id) do |job|
+          if job.release(self, pri, delay)
+            output("RELEASED\r\n")
+          end
+        end
+      end
+
+      def bury(stream, id, pri)
+        for_job(id) do |job|
+          if job.bury(self, pri)
+            output("BURIED\r\n")
+          end
+        end
+      end
+
+      def delete(stream, id)
+        for_job(id) do |job|
+          if job.delete(self)
+            output("DELETED\r\n")
+          end
+        end
+      end
+
+      def touch(stream, id)
+        for_job(id) do |job|
+          if job.touch(self)
+            output("TOUCHED\r\n")
+          end
+        end
+      end
+
+      def watch(stream, tube_name)
+        watched_tube_names << tube_name
+        watched_tube_names.uniq!
+        output_watching
+      end
+
+      def ignore(stream, tube_name)
+        watched_tube_names.delete(tube_name)
+        if watched_tube_names.empty?
+          watched_tube_names << tube_name
+          output("NOT_IGNORED\r\n")
+        else
+          output_watching
+        end
+      end
+
+      protected
+      def output_watching
+        output("WATCHING #{watched_tube_names.count}\r\n")
+      end
     end
   end
 end

data/lib/hydroponic_bean/connection.rb

@@ -4,8 +4,23 @@ module HydroponicBean
   class Connection
     include HydroponicBean::Protocol
 
+    attr_accessor :waiting
+    alias_method :waiting?, :waiting
+
     def initialize
       @_read, @_write = IO.pipe
+      @worker, @producer = false
+      @waiting = false
+      HydroponicBean.add_connection(self)
+    end
+
+    def worker?;   @worker;          end
+    def worker!;   @worker = true;   end
+    def producer?; @producer;        end
+    def producer!; @producer = true; end
+
+    def closed?
+      @_write.closed?
     end
 
     # Necessary interface used by beaneater
@@ -24,6 +39,7 @@ module HydroponicBean
     def close
       @_read.close
       @_write.close
+      HydroponicBean.remove_connection(self)
     end
 
     protected

data/lib/hydroponic_bean/data.rb

@@ -1,15 +1,56 @@
+require 'timeout'
+
 require 'hydroponic_bean/tube'
 require 'hydroponic_bean/job'
 
 module HydroponicBean
+  def self.clear
+    tubes.clear
+    jobs.clear
+    connections.clear
+    commands.clear
+  end
+
   def self.tubes
     @tubes ||= Hash.new{|h, k| h[k] = Tube.new(k)}
   end
 
+  def self.find_job(id)
+    id = id.to_i
+    if id == 0
+      return nil
+    else
+      job = jobs[id - 1]
+      job&.update_time!
+      return job
+    end
+  end
+
   def self.jobs
     @jobs ||= []
   end
 
+  def self.update_time!
+    jobs.each(&:update_time!)
+  end
+
+  def self.connections
+    @connections ||= []
+  end
+
+  def self.add_connection(connection)
+    connections.push(connection)
+  end
+
+  def self.remove_connection(connection)
+    connections.delete(connection)
+  end
+
+  # Keep track of commands for stats
+  def self.commands
+    @commands ||= Hash.new{|h, k| h[k] = 0}
+  end
+
   module Data
     def current_tube_name
       @current_tube_name ||= 'default'
@@ -23,12 +64,69 @@ module HydroponicBean
       @watched_tube_names ||= ['default']
     end
 
+    def watched_tubes
+      watched_tube_names.map do |name|
+        HydroponicBean.tubes[name]
+      end
+    end
+
     def create_job(pri, delay, ttr, data)
       job = Job.new(current_tube, pri, delay, ttr, data)
 
       HydroponicBean.jobs.push(job)
 
-      return job.id
+      return job
+    end
+
+    def deadline_soon?
+      HydroponicBean.update_time!
+      watched_tubes.map(&:reserved_jobs).flatten.select do |job|
+        job.reserved_by == self
+      end.sort_by(&:ttr_left).first&.deadline_soon?
+    end
+
+    def reserve_job
+      HydroponicBean.update_time!
+      reservable_jobs.first&.reserve(self)
+    end
+
+    def reservable_jobs
+      watched_tubes.reject(&:paused?).map(&:ready_jobs).flatten.sort_by(&:created_at).sort_by(&:pri)
+    end
+
+    def wait_for_job(timeout)
+      self.waiting = true
+      if timeout == 0
+        return reserve_job
+      else
+        timeout = [0, timeout].max
+        Timeout.timeout(timeout) do
+          while !(job = reserve_job)
+            sleep 0.49
+          end
+          return job
+        end
+      end
+    rescue Timeout::Error
+      return nil
+    ensure
+      self.waiting = false
+    end
+
+    def stats
+      {
+        'current-jobs-urgent' =>   HydroponicBean.jobs.select(&:urgent?).count,
+        'current-jobs-ready' =>    HydroponicBean.jobs.select(&:ready?).count,
+        'current-jobs-reserved' => HydroponicBean.jobs.select(&:reserved?).count,
+        'current-jobs-delayed' =>  HydroponicBean.jobs.select(&:delayed?).count,
+        'current-jobs-buried' =>   HydroponicBean.jobs.select(&:buried?).count,
+        'total-jobs' =>            HydroponicBean.jobs.count,
+        'current-tubes' =>         HydroponicBean.tubes.count,
+        'current-connections' =>   HydroponicBean.connections.count,
+        'current-producers' =>     HydroponicBean.connections.select(&:produced?).count,
+        'current-workers' =>       HydroponicBean.connections.select(&:workers?).count,
+        'current-waiting' =>       HydroponicBean.connections.select(&:waiting?).count,
+      }.merge(Hash[HydroponicBean.commands.map{|k, v| ["cmd-#{k}", v]}])
     end
   end
 end

data/lib/hydroponic_bean/job.rb

@@ -5,9 +5,7 @@ module HydroponicBean
     end
 
     attr_accessor :id, :pri, :delay, :ttr, :data, :created_at,
-      :state, :tube, :stats
-
-    attr_reader :deleted
+      :reserved_at, :reserved_by, :state, :tube, :stats
 
     def initialize(tube, pri, delay, ttr, data)
       @id = self.class.next_id
@@ -15,6 +13,7 @@ module HydroponicBean
       @tube = tube
       @pri = pri.to_i
       @delay = delay.to_i
+      @reserved_at = nil
       @ttr = ttr.to_i
       @state = @delay > 0 ? State.delayed : State.ready
       @data = data
@@ -30,6 +29,15 @@ module HydroponicBean
       @tube.push(self)
     end
 
+    def update_time!
+      if (delayed? || reserved?) && time_left == 0
+        if reserved?
+          stats['timeouts'] += 1
+        end
+        @state = State.ready
+      end
+    end
+
     def age
       (Time.now.utc - created_at).to_i
     end
@@ -44,13 +52,54 @@ module HydroponicBean
     def delayed?;  exists? && state == State.delayed; end
     def buried?;   exists? && state == State.buried; end
 
-    def delete
-      if @deleted == false
-        @tube.job_deleted
+    def reserved_by?(connection)
+      reserved? && reserved_by == connection
+    end
+
+    def reserve(connection)
+      if ready?
+        stats['reserves'] += 1
+        @state = State.reserved
+        @reserved_by = connection
+        @reserved_at = Time.now.utc
+        # For convenience and one-liners
+        return self
+      end
+    end
+
+    def release(connection, pri, delay)
+      if reserved_by?(connection)
+        stats['releases'] += 1
+        @pri = pri.to_i
+        @delay = delay.to_i
+        @reserved_at = nil
+        @reserved_by = nil
+        @state = @delay > 0 ? State.delayed : State.ready
+      end
+    end
+
+    def bury(connection, pri)
+      if reserved_by?(connection)
+        stats['buries'] += 1
+        @pri = pri.to_i
+        @reserved_at = nil
+        @reserved_by = nil
+        @state = State.buried
+      end
+    end
+
+    def delete(connection)
+      if exists? && (!reserved? || reserved_by?(connection))
+        @tube.job_deleted!
         @deleted = true
       end
     end
-    def exists?; !deleted; end
+
+    def touch(connection)
+      if reserved_by?(connection)
+        @reserved_at = Time.now.utc
+      end
+    end
 
     def kick
       if buried? || delayed?
@@ -59,8 +108,16 @@ module HydroponicBean
       end
     end
 
+    def ttr_left
+      [ttr - (Time.now.utc - reserved_at).to_i, 0].max
+    end
+
     def time_left
-      reserved? ? ttr : [delay - age, 0].max
+      reserved? ? ttr_left : [delay - age, 0].max
+    end
+
+    def deadline_soon?
+      ttr_left <= 1
     end
 
     def serialize_stats
@@ -77,6 +134,9 @@ module HydroponicBean
       }.merge(stats)
     end
 
+    def deleted?; @deleted; end
+    def exists?; !@deleted; end
+
     module State
       def self.ready;     :ready;    end
       def self.reserved;  :reserved; end