perfectqueue 0.8.54 → 0.9.0

data/lib/perfectqueue/command/perfectqueue.rb

@@ -8,7 +8,6 @@ op.banner += %[ <command>
 commands:
     list                             Show list of tasks
     submit <key> <type> <data>       Submit a new task
-    cancel_request <key>             Cancel request
     force_finish <key>               Force finish a task
     run <class>                      Run a worker process
     init                             Initialize a backend database
@@ -76,11 +75,6 @@ begin
     cmd = :list
     usage nil unless ARGV.length == 0
 
-  when 'cancel_request' ,'cancel'
-    cmd = :cancel
-    usage nil unless ARGV.length == 1
-    key = ARGV[0]
-
   when 'force_finish' ,'finish'
     cmd = :finish
     usage nil unless ARGV.length == 1
@@ -141,11 +135,6 @@ when :list
   }
   puts "#{n} entries."
 
-when :cancel
-  PerfectQueue.open(config_load_proc.call) {|queue|
-    queue[key].cancel_request!
-  }
-
 when :finish
   PerfectQueue.open(config_load_proc.call) {|queue|
     queue[key].force_finish!

data/lib/perfectqueue/multiprocess/child_process_monitor.rb

@@ -102,8 +102,8 @@ module PerfectQueue
       def send_signal(sig)
         begin
           Process.kill(sig, @pid)
-        rescue Errno::ESRCH, Errno::EPERM => e
-          @log.info "#{e.class}: #{e.message}\n#{e.backtrace}"
+        rescue Errno::ESRCH, Errno::EPERM
+          # TODO log?
         end
       end
 

data/lib/perfectqueue/multiprocess/thread_processor.rb

@@ -121,7 +121,7 @@ module PerfectQueue
       end
 
       def process(task)
-        @log.info "acquired task task=#{task.key} id=#{@processor_id}"
+        @log.info "acquired task task=#{task.key} id=#{@processor_id}: #{task.inspect}"
         begin
           r = @runner.new(task)
           @tm.set_task(task, r)

data/lib/perfectqueue/task.rb

@@ -27,10 +27,6 @@ module PerfectQueue
 
     attr_reader :key
 
-    def cancel_request!(options={})
-      @client.cancel_request(@key, options)
-    end
-
     def force_finish!(options={})
       @client.force_finish(@key, options)
     end

data/lib/perfectqueue/task_metadata.rb

@@ -68,10 +68,6 @@ module PerfectQueue
     def running?
       status == TaskStatus::RUNNING
     end
-
-    def cancel_requested?
-      status == TaskStatus::CANCEL_REQUESTED
-    end
   end
 
   class TaskMetadata

data/lib/perfectqueue/task_monitor.rb

@@ -147,7 +147,7 @@ module PerfectQueue
     def task_heartbeat
       @task.heartbeat!
     rescue
-      # finished, cancel_requested, preempted, etc.
+      # finished, preempted, etc.
       kill_task($!)
     end
   end
@@ -178,13 +178,6 @@ module PerfectQueue
       }
     end
 
-    def cancel_request!(*args, &block)
-      @log.info "cancel request task=#{self.key}" if @log
-      @task_monitor.task_finished(self) {
-        super(*args, &block)
-      }
-    end
-
     def update_data!(hash)
       @log.info "update data #{hash.inspect} task=#{self.key}" if @log
       @task_monitor.external_task_heartbeat(self) {

data/lib/perfectqueue/task_status.rb

@@ -21,7 +21,6 @@ module PerfectQueue
     WAITING = :waiting
     RUNNING = :running
     FINISHED = :finished
-    CANCEL_REQUESTED = :cancel_requested
   end
 end
 

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.54"
+  VERSION = "0.9.0"
 end

data/lib/perfectqueue/worker.rb

@@ -28,7 +28,6 @@ module PerfectQueue
       config = block.call
 
       @config = config
-      @log = config[:logger] || Logger.new(STDERR)
       @runner = runner
 
       @detach_wait = config[:detach_wait] || config['detach_wait'] || 10.0
@@ -67,11 +66,9 @@ module PerfectQueue
 
         else
           # child process finished unexpectedly
-          @log.info "Child process finished unexpectedly pid=#{pid}"
         end
 
-      rescue Errno::ECHILD => e
-        @log.info "#{e.class}: #{e.message}\n#{e.backtrace}"
+      rescue Errno::ECHILD
       end
     end
 
@@ -133,3 +130,4 @@ module PerfectQueue
   end
 
 end
+

data/spec/queue_spec.rb

@@ -20,8 +20,6 @@ describe Queue do
       queue.submit('task01', 'type1', {}, :now=>now+1)
     }.to raise_error AlreadyExistsError
 
-    queue['task01'].cancel_request!(:now=>now+2)
-
     expect {
       allow(STDERR).to receive(:puts)
       queue.submit('task01', 'type1', {}, :now=>now+10)
@@ -161,7 +159,6 @@ describe Queue do
     #queue['task01'].metadata.finished?.should == false
     #queue['task01'].metadata.running?.should == false
     #queue['task01'].metadata.waiting?.should == true
-    #queue['task01'].metadata.cancel_requested?.should == false
 
     task01 = queue.poll(:now=>now+10, :alive_time=>10)
     expect(task01.key).to eq('task01')
@@ -169,37 +166,12 @@ describe Queue do
     expect(queue['task01'].metadata.finished?).to eq(false)
     expect(queue['task01'].metadata.running?).to eq(true)
     expect(queue['task01'].metadata.waiting?).to eq(false)
-    expect(queue['task01'].metadata.cancel_requested?).to eq(false)
-
-    task01.cancel_request!
-
-    # status of cancel_requested running tasks is cancel_requested
-    expect(queue['task01'].metadata.finished?).to eq(false)
-    expect(queue['task01'].metadata.running?).to eq(false)
-    expect(queue['task01'].metadata.waiting?).to eq(false)
-    expect(queue['task01'].metadata.cancel_requested?).to eq(true)
 
     task01.finish!
 
     expect(queue['task01'].metadata.finished?).to eq(true)
     expect(queue['task01'].metadata.running?).to eq(false)
     expect(queue['task01'].metadata.waiting?).to eq(false)
-    expect(queue['task01'].metadata.cancel_requested?).to eq(false)
-  end
-
-  it 'fail canceling finished task' do
-    now = Time.now.to_i
-    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
-
-    task01 = queue.poll(:now=>now+10, :alive_time=>10)
-    expect(task01.key).to eq('task01')
-
-    task01.finish!
-
-    expect {
-      allow(STDERR).to receive(:puts)
-      queue['task01'].cancel_request!
-    }.to raise_error AlreadyFinishedError
   end
 
   it 'retention_time' do

data/spec/rdb_backend_spec.rb

@@ -21,11 +21,15 @@ describe Backend::RDBBackend do
 
   context '#submit' do
     it 'adds task' do
-      db.submit('key', '{"foo":"bar"}')
+      expect(db.submit('key', '{"foo":"bar"}')).to be true
       row = db.db.fetch("SELECT * FROM `#{table}` WHERE id=? LIMIT 1", 'key').first
       expect(row[:created_at]).not_to be_nil
       expect(row[:data]).to eq('{"foo":"bar"}')
     end
+    it 'returns nil for a duplicated task' do
+      expect(db.submit('key', '{"foo":"bar"}')).to be true
+      expect(db.submit('key', '{"foo":"bar"}')).to be_nil
+    end
   end
 
   context '#cancel' do
@@ -55,7 +59,7 @@ describe Backend::RDBBackend do
     end
     context 'error' do
       it 'returns block result' do
-        expect(RuntimeError).to receive(:new).exactly(db.max_retry_count).and_call_original
+        expect(RuntimeError).to receive(:new).exactly(Backend::RDBBackend::MAX_RETRY).and_call_original
         allow(STDERR).to receive(:puts)
         allow(db).to receive(:sleep)
         expect do
@@ -65,19 +69,5 @@ describe Backend::RDBBackend do
         end.to raise_error(RuntimeError)
       end
     end
-    context 'cannot connect' do
-      let (:uri){ 'mysql2://root:@nonexistent/perfectqueue_test' }
-      let (:db) do
-        Backend::RDBBackend.new(uri, table)
-      end
-      it 'raises Sequel::DatabaseConnectionError' do
-        allow(STDERR).to receive(:puts)
-        slept = 0
-        expect(db).to receive(:sleep).exactly(9).times{|n| slept += n }
-        expect(db.db).to receive(:connect).exactly(10).times.and_call_original
-        expect{ db.__send__(:connect){ db.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
-        expect(slept).to be < 30
-      end
-    end
   end
 end

data/spec/rdb_compat_backend_spec.rb

@@ -166,11 +166,15 @@ describe Backend::RDBCompatBackend do
   end
 
   context '#submit' do
-    it 'adds task' do
-      db.submit('key', 'test', nil, {})
+    it 'returns true' do
+      expect(db.submit('key', 'test', nil, {})).to be_an_instance_of(Task)
+    end
+    it 'returns true (gzip)' do
+      expect(db.submit('key', 'test', nil, {compression: 'gzip'})).to be_an_instance_of(Task)
     end
-    it 'gzip' do
-      db.submit('key', 'test', nil, {compression: 'gzip'})
+    it 'returns nil if duplication' do
+      expect(db.submit('key', 'test', nil, {})).to be_an_instance_of(Task)
+      expect{db.submit('key', 'test', nil, {})}.to raise_error(IdempotentAlreadyExistsError)
     end
   end
 
@@ -195,85 +199,52 @@ describe Backend::RDBCompatBackend do
         expect(ary[0]).to be_an_instance_of(AcquiredTask)
       end
     end
-    context 'some tasks' do
-      let :t0 do now - 100 end
-      before do
-        db.submit('key1', 'test1', nil, {now: t0})
-        db.submit('key2', 'test2', nil, {now: t0})
-        db.submit('key3', 'test3', nil, {now: t0})
+    context 'disable_resource_limit' do
+      let (:config) do
+        {url: 'mysql2://root:@localhost/perfectqueue_test', table: table, disable_resource_limit: true}
       end
-      it 'returns 3 tasks' do
-        now0 = Time.at(t0)
-        expect(now0).to receive(:to_time).exactly(3).times.and_call_original
-        db.list({}) do |task|
-          expect(task.timeout).to eq now0.to_time
-        end
-        ary = db.acquire(alive_time, max_acquire, {now: now})
-        expect(ary).to be_an_instance_of(Array)
-        expect(ary.size).to eq(3)
-        expect(ary[0]).to be_an_instance_of(AcquiredTask)
-        expect(ary[1]).to be_an_instance_of(AcquiredTask)
-        expect(ary[2]).to be_an_instance_of(AcquiredTask)
-
-        now1 = Time.at(now + alive_time)
-        expect(now1).to receive(:to_time).exactly(3).times.and_call_original
-        db.list({}){|task| expect(task.timeout).to eq now1.to_time }
+      before do
+        db.submit(key, 'test', nil, {})
       end
-      it 'returns 2 tasks' do
-        db.instance_variable_set(:@prefetch_break_types, 'test2')
+      it 'returns a task' do
         ary = db.acquire(alive_time, max_acquire, {})
         expect(ary).to be_an_instance_of(Array)
-        expect(ary.size).to eq(2)
+        expect(ary.size).to eq(1)
         expect(ary[0]).to be_an_instance_of(AcquiredTask)
-        expect(ary[1]).to be_an_instance_of(AcquiredTask)
       end
     end
-    context 'stole a task' do
-      let :t0 do now - 100 end
+    context 'some tasks' do
+      let :t0 do now - 300 end
+      let :t1 do now - 200 end
+      let :t2 do now - 100 end
       before do
         db.submit('key1', 'test1', nil, {now: t0})
         db.submit('key2', 'test2', nil, {now: t0})
-        db.submit('key3', 'test3', nil, {now: t0})
-      end
-      it 'returns nil' do
-        # hook and steal a task
-        mock = double('prefetch_break_types')
-        db.instance_variable_set(:@prefetch_break_types, mock)
-        allow(mock).to receive(:include?) do
-          db.db['UPDATE `test_queues` SET timeout=? WHERE id=?', now+300, 'key2'].update
-          false
-        end
+        db.submit('key3', 'test3', nil, {now: t1})
+        db.submit('key4', 'test4', nil, {now: t2})
+        db.submit('key5', 'test5', nil, {now: t2})
+      end
+      it 'returns 5 tasks' do
+        ary = []
+        db.list({}){|task| ary << task }
+        expect(ary[0].timeout.to_i).to eq t0
+        expect(ary[1].timeout.to_i).to eq t0
+        expect(ary[2].timeout.to_i).to eq t1
+        expect(ary[3].timeout.to_i).to eq t2
+        expect(ary[4].timeout.to_i).to eq t2
 
-        ary = db.acquire(alive_time, max_acquire, {})
-        expect(ary).to be_nil
-      end
-    end
-  end
+        ary = db.acquire(alive_time, max_acquire, {now: now})
+        expect(ary).to be_an_instance_of(Array)
+        expect(ary.size).to eq(5)
+        expect(ary[0]).to be_an_instance_of(AcquiredTask)
+        expect(ary[1]).to be_an_instance_of(AcquiredTask)
+        expect(ary[2]).to be_an_instance_of(AcquiredTask)
+        expect(ary[3]).to be_an_instance_of(AcquiredTask)
+        expect(ary[4]).to be_an_instance_of(AcquiredTask)
 
-  context '#cancel_request' do
-    let (:key){ 'key' }
-    let (:task_token){ Backend::RDBCompatBackend::Token.new(key) }
-    let (:retention_time) { 42 }
-    let (:delete_timeout){ now + retention_time }
-    let (:options){ {now: now} }
-    context 'have a task' do
-      before do
-        db.submit(key, 'test', nil, {})
-      end
-      it 'returns nil' do
-        expect(db.cancel_request(key, options)).to be_nil
-        row = db.db.fetch("SELECT created_at FROM `#{table}` WHERE id=? LIMIT 1", key).first
-        expect(row[:created_at]).to eq 0
-      end
-    end
-    context 'already finished' do
-      before do
-        expect(STDERR).to receive(:puts)
-        db.submit(key, 'test', nil, {})
-        db.finish(task_token, retention_time, options)
-      end
-      it 'raises AlreadyFinishedError' do
-        expect{db.cancel_request(key, options)}.to raise_error(AlreadyFinishedError)
+        now1 = Time.at(now + alive_time)
+        expect(now1).to receive(:to_time).exactly(5).times.and_call_original
+        db.list({}){|task| expect(task.timeout).to eq now1.to_time }
       end
     end
   end
@@ -348,15 +319,6 @@ describe Backend::RDBCompatBackend do
         expect{db.heartbeat(task_token, 0, {})}.to raise_error(PreemptedError)
       end
     end
-    context 'canceled task' do
-      before do
-        db.submit(key, 'test', nil, {})
-        db.cancel_request(key, options)
-      end
-      it 'returns nil' do
-        expect( db.heartbeat(task_token, 0, {}) ).to be_nil
-      end
-    end
   end
 
   context '#connect' do
@@ -367,7 +329,7 @@ describe Backend::RDBCompatBackend do
     end
     context 'error' do
       it 'returns block result' do
-        expect(RuntimeError).to receive(:new).exactly(db.max_retry_count).and_call_original
+        expect(RuntimeError).to receive(:new).exactly(Backend::RDBCompatBackend::MAX_RETRY).and_call_original
         allow(STDERR).to receive(:puts)
         allow(db).to receive(:sleep)
         expect do
@@ -376,18 +338,6 @@ describe Backend::RDBCompatBackend do
           end
         end.to raise_error(RuntimeError)
       end
-      context 'cannot connect' do
-        let (:config){ {url: 'mysql2://root:@nonexistent/perfectqueue_test', table: table} }
-        it 'raises Sequel::DatabaseConnectionError' do
-          allow(STDERR).to receive(:puts)
-          d = Backend::RDBCompatBackend.new(client, config)
-          slept = 0
-          expect(d).to receive(:sleep).exactly(9).times{|n| slept += n }
-          expect(d.db).to receive(:connect).exactly(10).times.and_call_original
-          expect{ d.__send__(:connect){ d.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
-          expect(slept).to eq(18)
-        end
-      end
     end
   end
 
@@ -541,13 +491,6 @@ describe Backend::RDBCompatBackend do
         end
       end
     end
-    context 'created_at is 0' do
-      it 'status is :cancel_requested' do
-        data[:created_at] = 0
-        r = db.__send__(:create_attributes, now, row)
-        expect(r[:status]).to eq TaskStatus::CANCEL_REQUESTED
-      end
-    end
     context 'created_at > 0' do
       context 'timeout' do
         it 'status is :waiting' do

data/spec/spec_helper.rb

@@ -1,16 +1,15 @@
 $LOAD_PATH.unshift(File.expand_path('../lib', File.dirname(__FILE__)))
 
-require 'perfectqueue'
-
 if ENV['SIMPLE_COV']
   require 'simplecov'
   SimpleCov.start do
-    add_filter 'spec/'
     add_filter 'pkg/'
     add_filter 'vendor/'
   end
 end
 
+require 'perfectqueue'
+
 if ENV["CI"]
   require 'coveralls'
   Coveralls.wear!