tuber 0.0.1 → 0.5.1

data/test/jobs_test.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+# test/jobs_test.rb
+
+require File.expand_path('../test_helper', __FILE__)
+
+describe Tuber::Jobs do
+  before do
+    @beanstalk = Tuber.new(tuber_address)
+    @jobs = Tuber::Jobs.new(@beanstalk)
+    @tube = @beanstalk.tubes.find('baz')
+  end
+
+  describe "for #find" do
+    before do
+      @time = Time.now.to_i
+      @tube.put("foo find #{@time}")
+      @job = @tube.peek(:ready)
+    end
+
+    it "should return job from id" do
+      assert_equal "foo find #{@time}", @jobs.find(@job.id).body
+    end
+
+    it "should return job using peek" do
+      assert_equal "foo find #{@time}", @jobs.find(@job.id).body
+    end
+
+    it "should return job using hash syntax" do
+      assert_equal "foo find #{@time}", @jobs.find(@job.id).body
+    end
+
+    it "should return nil for invalid negative id" do
+      assert_nil @jobs.find(10000) 
+    end
+
+    it "should return nil for invalid negative id" do
+      assert_raises(Tuber::BadFormatError) { @jobs.find(-1) }
+    end
+  end # find
+
+  describe "for #touch_all" do
+    before do
+      require_tuber!
+      @touch_tube = @beanstalk.tubes.find('touch_all_tube')
+      @beanstalk.tubes.watch! 'touch_all_tube'
+    end
+
+    after do
+      cleanup_tubes!(['touch_all_tube'], @beanstalk)
+    end
+
+    it "should return 0 when no jobs are held" do
+      assert_equal 0, @beanstalk.jobs.touch_all
+    end
+
+    it "should count every job held by the connection" do
+      3.times { |i| @touch_tube.put "touch all #{i}", :ttr => 5 }
+      jobs = @beanstalk.tubes.reserve_batch(3)
+      assert_equal 3, jobs.size
+      assert_equal 3, @beanstalk.jobs.touch_all
+      jobs.each(&:delete)
+    end
+
+    it "should exclude jobs the worker no longer holds" do
+      2.times { |i| @touch_tube.put "touch all drop #{i}", :ttr => 5 }
+      jobs = @beanstalk.tubes.reserve_batch(2)
+      jobs.first.delete
+      assert_equal 1, @beanstalk.jobs.touch_all
+      jobs.last.delete
+      assert_equal 0, @beanstalk.jobs.touch_all
+    end
+
+    it "should extend the ttr of held jobs" do
+      @touch_tube.put "touch all ttr", :ttr => 3
+      job = @beanstalk.tubes.reserve
+      sleep 2
+      assert_equal 1, @beanstalk.jobs.touch_all
+      sleep 2
+      assert_equal 'reserved', job.stats.state
+      job.delete
+    end
+  end # touch_all
+
+  describe "for #register!" do
+    before do
+      $foo = 0
+      @jobs.register('tube', :retry_on => [Timeout::Error]) do |job|
+        $foo += 1
+      end
+    end
+
+    it "should store processor" do
+      assert_equal 'tube', @jobs.processors.keys.first
+      assert_equal [Timeout::Error], @jobs.processors.values.first[:retry_on]
+    end
+
+    it "should store block for 'tube'" do
+      @jobs.processors['tube'][:block].call nil
+      assert_equal 1, $foo
+    end
+  end # register!
+
+  describe "for process!" do
+    before do
+      $foo = []
+
+      @jobs.register('tube_success', :retry_on => [Timeout::Error]) do |job|
+        # p job.body
+        $foo << job.body
+        raise Tuber::AbortProcessingError if job.body =~ /abort/
+      end
+
+      @jobs.register('tube_release', :retry_on => [Timeout::Error], :max_retries => 2) do |job|
+        $foo << job.body
+        raise Timeout::Error
+      end
+
+      @jobs.register('tube_buried') do |job|
+        $foo << job.body
+        raise RuntimeError
+      end
+
+      cleanup_tubes!(['tube_success', 'tube_release', 'tube_buried'])
+
+      @beanstalk.tubes.find('tube_success').put("success abort", :pri => 2**31 + 1)
+      @beanstalk.tubes.find('tube_success').put("success 2", :pri => 1)
+      @beanstalk.tubes.find('tube_release').put("released")
+      @beanstalk.tubes.find('tube_buried').put("buried")
+
+      @jobs.process!(:release_delay => 0)
+    end
+
+    it "should process all jobs" do
+      assert_equal ['success 2', 'released', 'released', 'released', 'buried', 'success abort'], $foo
+    end
+
+    it "should clear successful_jobs" do
+      assert_equal 0, @beanstalk.tubes.find('tube_success').stats.current_jobs_ready
+      assert_equal 1, @beanstalk.tubes.find('tube_success').stats.current_jobs_buried
+      assert_equal 0, @beanstalk.tubes.find('tube_success').stats.current_jobs_reserved
+    end
+
+    it "should retry release jobs 2 times" do
+      assert_equal 2, @beanstalk.tubes.find('tube_release').peek(:buried).stats.releases
+    end
+
+    it "should bury unexpected exception" do
+      assert_equal 1, @beanstalk.tubes.find('tube_buried').stats.current_jobs_buried
+    end
+  end # for_process!
+end # Tuber::Jobs

data/test/prompt_regexp_test.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# test/prompt_regexp_test.rb
+
+require File.expand_path('../test_helper', __FILE__)
+require 'socket'
+
+describe "Reading from socket client" do
+  before do
+    @fake_port = 11301
+    @tube_name = 'tube.to.test'
+
+    @fake_server = Thread.start do
+      server = TCPServer.new(@fake_port)
+      loop do
+        IO.select([server])
+        client = server.accept_nonblock
+        while line = client.gets
+          case line
+          when /list-tubes-watched/i
+            client.print "OK #{7+@tube_name.size}\r\n---\n- #{@tube_name}\n\r\n"
+          when /watch #{@tube_name}/i
+            client.print "WATCHING 1\r\n"
+          when /reserve/i
+            client.print "RESERVED 17 25\r\n"
+            client.print "[first part]"
+            # Emulate network delay
+            sleep 0.5
+            client.print "[second part]\r\n"
+          else
+            client.print "ERROR\r\n"
+          end
+        end
+      end
+    end
+
+    slept = 0
+    while @beanstalk.nil?
+      begin
+        @beanstalk = Tuber.new("localhost:#{@fake_port}")
+      rescue Tuber::NotConnected
+        raise 'Could not connect to fake beanstalkd server' if slept > 1
+        sleep 0.1
+        slept += 0.1
+      end
+    end
+
+  end
+
+  it 'should reserve job with full body' do
+    job = @beanstalk.tubes[@tube_name].reserve
+    assert_equal '[first part][second part]', job.body
+  end
+
+  after do
+    @beanstalk.close
+    @fake_server.kill
+  end
+end

data/test/stat_struct_test.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# test/stat_struct_test.rb
+
+require File.expand_path('../test_helper', __FILE__)
+
+describe Tuber::StatStruct do
+  before do
+    @hash = { :foo => "bar", :bar => "baz", :baz => "foo", :"under-score" => "demo" }
+    @struct = Tuber::StatStruct.from_hash(@hash)
+  end
+
+  describe "for #from_hash" do
+    it "should have 4 keys" do
+      assert_equal 'bar', @struct.foo
+      assert_equal 'baz', @struct.bar
+      assert_equal 'foo', @struct.baz
+      assert_equal 'demo', @struct.under_score
+    end
+  end # from_hash
+
+  describe "for [] access" do
+    it "should have hash lookup" do
+      assert_equal 'bar', @struct['foo']
+      assert_equal 'baz', @struct['bar']
+    end
+
+    it "should convert keys to string" do
+      assert_equal 'foo', @struct[:baz]
+      assert_equal 'demo', @struct[:"under_score"]
+    end
+
+    it "should look up hyphenated beanstalkd keys" do
+      assert_equal 'demo', @struct['under-score']
+      assert_equal 'demo', @struct[:"under-score"]
+    end
+  end # []
+
+  describe "for #keys" do
+    it "should return 4 keys" do
+      assert_equal 4, @struct.keys.size
+    end
+
+    it "should return expected keys" do
+      assert_equal ['foo', 'bar', 'baz', 'under_score'].sort, @struct.keys.sort
+    end
+  end # keys
+
+  describe "for #to_h" do
+    it "should return 4 keys / values" do
+      assert_equal 4, @struct.to_h.size
+    end
+
+    it "should return expect hash" do
+      assert_equal [['foo', 'bar'], ['bar', 'baz'], ['baz', 'foo'], ['under_score', 'demo']].sort, @struct.to_h.sort
+    end
+  end # to_h
+end # Tuber::StatStruct

data/test/stats_test.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# test/stats_test.rb
+
+require File.expand_path('../test_helper', __FILE__)
+
+describe Tuber::Stats do
+  before do
+    connection = stub(:transmit => { :body => { 'uptime' => 1, 'cmd-use' => 2 }, :status => "OK"})
+    @beanstalk = stub(:connection => connection)
+    @stats = Tuber::Stats.new(@beanstalk)
+  end
+
+  describe 'for #[]' do
+    it "should return stats by key" do
+      assert_equal 1, @stats[:uptime]
+    end
+
+    it "should return stats by underscore key" do
+      assert_equal 2, @stats[:'cmd_use']
+    end
+  end # []
+
+  describe 'for #keys' do
+    it "should return list of keys" do
+      assert_equal 2, @stats.keys.size
+      assert @stats.keys.include?('uptime'), "Expected keys to include 'uptime'"
+      assert @stats.keys.include?('cmd_use'), "Expected keys to include 'cmd-use'"
+    end
+  end # keys
+
+  describe 'for #method_missing' do
+    it "should return stats by key" do
+      assert_equal 1, @stats.uptime
+    end
+
+    it "should return stats by underscore key" do
+      assert_equal 2, @stats.cmd_use
+    end
+
+    it "should raise NoMethodError" do
+      assert_raises(NoMethodError) { @stats.cmd }
+    end
+  end # method_missing
+end # Tuber::Stats

data/test/test_helper.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+ENV["TEST"] = 'true'
+require 'rubygems'
+require 'coveralls'
+Coveralls.wear!
+require 'minitest/autorun'
+$:.unshift File.expand_path("../../lib")
+require 'tuber'
+require 'timeout'
+begin
+  require 'mocha/minitest'
+rescue LoadError
+  require 'mocha'
+end
+require 'json'
+
+class MiniTest::Unit::TestCase
+
+  # Address of the server the integration-style tests talk to. Override with
+  # TUBER_TEST_ADDRESS when localhost:11300 isn't the server you mean
+  # (e.g. TUBER_TEST_ADDRESS=127.0.0.1 when another queue holds the IPv6 port).
+  def tuber_address
+    ENV.fetch('TUBER_TEST_ADDRESS', 'localhost')
+  end
+
+  # True when the server under test is Tuber rather than stock beanstalkd.
+  def tuber_server?
+    if $tuber_server.nil?
+      client = Tuber.new(tuber_address)
+      begin
+        $tuber_server = client.connection.transmit('stats')[:body]['version'].to_s.include?('tuber')
+      ensure
+        client.close
+      end
+    end
+    $tuber_server
+  end
+
+  # Skips the current test unless the server speaks Tuber's extended protocol
+  # (reserve-batch, touch-all, flush-buried, ...).
+  def require_tuber!
+    skip "requires a Tuber server, not stock beanstalkd" unless tuber_server?
+  end
+
+  # Cleans up all jobs from specific tubes
+  #
+  # @example
+  #   cleanup_tubes!(['foo'], @beanstalk)
+  #
+  def cleanup_tubes!(tubes, client=nil)
+    client ||= @beanstalk
+    tubes.each do |name|
+      client.tubes.find(name).clear
+    end
+  end
+
+  # Cleans up all jobs from all tubes known to the connection
+  def flush_all(client=nil)
+    client ||= @beanstalk
+
+    # Do not continue if it is a mock or the connection has been closed
+    return if !client.is_a?(Tuber) || !client.connection.connection
+
+    client.tubes.all.each do |tube|
+      tube.clear
+    end
+  end
+
+  # Run clean up after each test to ensure clean state in all tests
+  def teardown
+    flush_all
+  end
+end

data/test/tube_test.rb

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+# test/tube_test.rb
+
+require File.expand_path('../test_helper', __FILE__)
+
+describe Tuber::Tube do
+  before do
+    @beanstalk = Tuber.new(tuber_address)
+    @tube = Tuber::Tube.new(@beanstalk, 'baz')
+  end
+
+  describe "for #put" do
+    before do
+      @time = Time.now.to_i
+    end
+
+    it "should insert a job" do
+      @tube.put "bar put #{@time}"
+      assert_equal "bar put #{@time}", @tube.peek(:ready).body
+      assert_equal 120, @tube.peek(:ready).ttr
+      assert_equal 65536, @tube.peek(:ready).pri
+      assert_equal 0, @tube.peek(:ready).delay
+    end
+
+    it "should insert a delayed job" do
+      @tube.put "delayed put #{@time}", :delay => 1
+      assert_equal "delayed put #{@time}", @tube.peek(:delayed).body
+    end
+
+    it "should support custom serializer" do
+      Tuber.configure.job_serializer = lambda { |b| JSON.dump(b) }
+      @tube.put({ foo: "bar"})
+      assert_equal '{"foo":"bar"}', @tube.peek(:ready).body
+    end
+
+    after do
+      Tuber::Connection.any_instance.unstub(:transmit)
+      Tuber.configure.job_serializer = lambda { |b| b }
+    end
+  end # put
+
+  describe "for #peek" do
+    before do
+      @time = Time.now.to_i
+    end
+
+    it "should peek delayed" do
+      @tube.put "foo delay #{@time}", :delay => 1
+      assert_equal "foo delay #{@time}", @tube.peek(:delayed).body
+    end
+
+    it "should peek ready" do
+      @tube.put "foo ready #{@time}", :delay => 0
+      assert_equal "foo ready #{@time}", @tube.peek(:ready).body
+    end
+
+    it "should peek buried" do
+      @tube.put "foo buried #{@time}"
+      @tube.reserve.bury
+
+      assert_equal "foo buried #{@time}", @tube.peek(:buried).body
+    end
+
+    it "should return nil for empty peek" do
+      assert_nil @tube.peek(:buried)
+    end
+
+    it "supports valid JSON" do
+      json = '{ "foo" : "bar" }'
+      @tube.put(json)
+      assert_equal 'bar', JSON.parse(@tube.peek(:ready).body)['foo']
+    end
+
+    it "supports non valid JSON" do
+      json = '{"message":"hi"}'
+      @tube.put(json)
+      assert_equal 'hi', JSON.parse(@tube.peek(:ready).body)['message']
+    end
+
+    it "supports passing crlf through" do
+      @tube.put("\r\n")
+      assert_equal "\r\n", @tube.peek(:ready).body
+    end
+
+    it "supports passing any byte value through" do
+      bytes = (0..255).to_a.pack("c*")
+      @tube.put(bytes)
+      assert_equal bytes, @tube.peek(:ready).body
+    end
+
+    it "should support custom parser" do
+      Tuber.configure.job_parser = lambda { |b| JSON.parse(b) }
+      json = '{"message": "hi: there! this: is a long message"}'
+      @tube.put(json)
+      assert_equal 'hi: there! this: is a long message', @tube.peek(:ready).body['message']
+    end
+
+    after do
+      Tuber.configure.job_parser = lambda { |b| b }
+    end
+  end # peek
+
+  describe "for #reserve" do
+    before do
+      @time = Time.now.to_i
+      @json = %Q&{ "foo" : "#{@time} bar" }&
+      @tube.put @json
+    end
+
+    it "should reserve job" do
+      @job = @tube.reserve
+      assert_equal "#{@time} bar", JSON.parse(@job.body)['foo']
+      @job.delete
+    end
+
+    it "should reserve job with block" do
+      job = nil
+      @tube.reserve { |j| job = j; job.delete }
+      assert_equal "#{@time} bar", JSON.parse(job.body)['foo']
+    end
+
+    it "should support custom parser" do
+      Tuber.configure.job_parser = lambda { |b| JSON.parse(b) }
+      @job = @tube.reserve
+      assert_equal "#{@time} bar", @job.body['foo']
+      @job.delete
+    end
+
+    after do
+      Tuber.configure.job_parser = lambda { |b| b }
+    end
+  end # reserve
+
+  describe "for #put with tuber tags" do
+    before do
+      @beanstalk = stub
+      @connection = stub(tube_used: 'baz')
+      @beanstalk.stubs(:connection).returns(@connection)
+      @tubes = stub
+      @tubes.stubs(:use).with('baz').returns('baz')
+      @beanstalk.stubs(:tubes).returns(@tubes)
+      @tube = Tuber::Tube.new(@beanstalk, 'baz')
+    end
+
+    it "should append idp tag as string" do
+      @connection.expects(:transmit).with("put 65536 0 120 4 idp:report\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data", idp: "report"
+    end
+
+    it "should append idp tag with ttl as array" do
+      @connection.expects(:transmit).with("put 65536 0 120 4 idp:report:300\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data", idp: ["report", 300]
+    end
+
+    it "should append con tag as string" do
+      @connection.expects(:transmit).with("put 65536 0 120 4 con:db\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data", con: "db"
+    end
+
+    it "should append con tag with limit as array" do
+      @connection.expects(:transmit).with("put 65536 0 120 4 con:db:3\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data", con: ["db", 3]
+    end
+
+    it "should append grp tag" do
+      @connection.expects(:transmit).with("put 65536 0 120 4 grp:batch-1\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data", grp: "batch-1"
+    end
+
+    it "should append aft tag" do
+      @connection.expects(:transmit).with("put 65536 0 120 4 aft:batch-0\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data", aft: "batch-0"
+    end
+
+    it "should append all four tags" do
+      @connection.expects(:transmit).with("put 65536 0 120 4 idp:key:60 con:db:2 grp:g1 aft:g0\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data", idp: ["key", 60], con: ["db", 2], grp: "g1", aft: "g0"
+    end
+
+    it "should send plain put without tags" do
+      @connection.expects(:transmit).with("put 65536 0 120 4\r\ndata").returns({status: "INSERTED", id: "1"})
+      @tube.put "data"
+    end
+  end # put with tuber tags
+
+  describe "for #pause" do
+    before do
+      @time = Time.now.to_i
+      @tube = Tuber::Tube.new(@beanstalk, 'bam')
+      @tube.put "foo pause #{@time}"
+    end
+
+    it "should allow tube pause" do
+      assert_equal 0, @tube.stats.pause
+      @tube.pause(1)
+      assert_equal 1, @tube.stats.pause
+    end
+  end # pause
+
+  describe "for #stats" do
+    before do
+      @time = Time.now.to_i
+      @tube.put "foo stats #{@time}"
+      @stats = @tube.stats
+    end
+
+    it "should return total number of jobs in tube" do
+      assert_equal 1, @stats['current_jobs_ready']
+      assert_equal 0, @stats.current_jobs_delayed
+    end
+
+    it "should raise error for empty tube" do
+      assert_raises(Tuber::NotFoundError) { @beanstalk.tubes.find('fake_tube').stats }
+    end
+  end # stats
+
+  describe "for #kick" do
+    before do
+      @time, @time2 = 2.times.map { Time.now.to_i }
+      @tube.put "kick #{@time}"
+      @tube.put "kick #{@time2}"
+
+      2.times.map { @tube.reserve.bury }
+    end
+
+    it "should kick 2 buried jobs" do
+      assert_equal 2, @tube.stats.current_jobs_buried
+      @tube.kick(2)
+      assert_equal 0, @tube.stats.current_jobs_buried
+      assert_equal 2, @tube.stats.current_jobs_ready
+    end
+  end # kick
+
+  describe "for #clear" do
+    @time = Time.now.to_i
+    before do
+      2.times { |i| @tube.put "to clear success #{i} #{@time}" }
+      2.times { |i| @tube.put "to clear delayed #{i} #{@time}", :delay => 5 }
+      2.times { |i| @tube.put "to clear bury #{i} #{@time}", :pri => 1 }
+      @tube.reserve.bury while @tube.peek(:ready).stats['pri'] == 1
+    end
+
+    it "should clear all jobs in tube" do
+      tube_counts = lambda { %w(ready buried delayed).map { |s| @tube.stats["current_jobs_#{s}"] } }
+      assert_equal [2, 2, 2], tube_counts.call
+      @tube.clear
+      @tube.stats
+      assert_equal [0, 0, 0], tube_counts.call
+    end
+  end # clear
+  describe "for #flush" do
+    before do
+      require_tuber!
+      @tube = Tuber::Tube.new(@beanstalk, 'flush_test')
+      3.times { |i| @tube.put "flush job #{i}" }
+    end
+
+    it "should flush all jobs and return count" do
+      count = @tube.flush
+      assert_equal 3, count
+      assert_nil @tube.peek(:ready)
+    end
+
+    it "should return 0 for empty tube" do
+      @tube.flush
+      count = @tube.flush
+      assert_equal 0, count
+    end
+  end # flush
+
+  describe "for #flush_buried" do
+    before do
+      require_tuber!
+      @tube = Tuber::Tube.new(@beanstalk, 'flush_buried_test')
+      @beanstalk.tubes.watch! 'flush_buried_test'
+    end
+
+    after do
+      @tube.clear
+    end
+
+    it "should flush only buried jobs and return count" do
+      2.times { |i| @tube.put "buried job #{i}" }
+      2.times { @beanstalk.tubes.reserve.bury }
+      @tube.put "ready job"
+
+      count = @tube.flush_buried
+      assert_equal 2, count
+      assert_nil @tube.peek(:buried)
+      refute_nil @tube.peek(:ready)
+    end
+
+    it "should return 0 when there are no buried jobs" do
+      @tube.put "ready only"
+      assert_equal 0, @tube.flush_buried
+    end
+  end # flush_buried
+end # Tuber::Tube