perfectqueue 0.8.44.1 → 0.8.45

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff48a49e577eab839217d96f1efbe4c3bbb74174
-  data.tar.gz: d7ddae1782d03ffccacfab5143d7dc08a2f33c1c
+  metadata.gz: ca67d530351a8a7e0ff2f950d10248bc23eb5e2e
+  data.tar.gz: 5b66bc3219182f721af358e0bce7836ffdca356d
 SHA512:
-  metadata.gz: 59b93afc56e199edaee68619425bbba22bc63674f0f4ae4fe2b7dd6c1bde59e2c08cee771a0e60f1020c3d10a139b39e8fa7328c2714bc978d6220dd516d97de
-  data.tar.gz: 58ba8303acadf21474055d62c9cab9dc4fdc734987f709e7063ba448e7240309eb7b83e63bae0c2ec10767a18561acb14e093443a028788f1eafe30e5ee16063
+  metadata.gz: 282b4e05105a5b6fe92cfb3c406b415221784350cb1449e87fe42b481fa7adc0b5d5318c67cc5922e9deafbd244939b2481e76c10ced8b15e981605eacddde35
+  data.tar.gz: 7f4a15e96b072fa04144d04f6f92396a730b76de07c0cb45c6f63d9c30d84bbfff88d7cab526a7a895d6265ed74cc897f9d0b8293d3a7c8271d67cba65c4acee

data/ChangeLog

@@ -1,6 +1,9 @@
-== 2016-06-14 version 0.8.44.1
+== 2015-12-02 version 0.8.45
 
-* Don't delete v0.8.46's finished tasks for migration
+* remove sqlite3 support
+* fix #3
+* add specs
+* fix other trivial bugs
 
 == 2015-10-07 version 0.8.44
 

data/lib/perfectqueue/application/decider.rb

@@ -19,7 +19,7 @@
 module PerfectQueue
   module Application
 
-    class UndefinedDecisionError
+    class UndefinedDecisionError < StandardError
     end
 
     class Decider
@@ -39,7 +39,7 @@ module PerfectQueue
         begin
           m = method(type)
         rescue NameError
-          raise UndefinedDecisionError, "Undefined decision #{type} options=#{opt.inspect}"
+          raise UndefinedDecisionError, "Undefined decision #{type} options=#{opts.inspect}"
         end
         m.call(opts)
       end

data/lib/perfectqueue/application/router.rb

@@ -58,7 +58,7 @@ module PerfectQueue
         when String, Symbol
           pattern = /\A#{Regexp.escape(pattern)}\z/
         else
-          raise ArguementError, "pattern should be String or Regexp but got #{pattern.class}: #{pattern.inspect}"
+          raise ArgumentError, "pattern should be String or Regexp but got #{pattern.class}: #{pattern.inspect}"
         end
 
         @patterns << [pattern, sym]

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -21,9 +21,6 @@ module PerfectQueue
     class RDBCompatBackend
       include BackendHelper
 
-      # backport from v0.8.46
-      EVENT_HORIZON = 13_0000_0000 # 2011-03-13 07:06:40 UTC
-
       class Token < Struct.new(:key)
       end
 
@@ -40,10 +37,7 @@ module PerfectQueue
         #password = config[:password]
         #user = config[:user]
 
-        case url.split('//',2)[0].to_s
-        when /sqlite/i
-          @db = Sequel.connect(url, :max_connections=>1)
-        when /mysql/i
+        if /\Amysql/i =~ url
           require 'uri'
 
           uri = URI.parse(url)
@@ -63,8 +57,20 @@ module PerfectQueue
           else
             @use_connection_pooling = !!config[:sslca]
           end
+          @table_lock = lambda {
+            locked = nil
+            loop do
+              @db.fetch("SELECT GET_LOCK('#{@table}', #{LOCK_WAIT_TIMEOUT}) locked") do |row|
+                locked = true if row[:locked] == 1
+              end
+              break if locked
+            end
+          }
+          @table_unlock = lambda {
+            @db.run("SELECT RELEASE_LOCK('#{@table}')")
+          }
         else
-          raise ConfigError, "'sqlite' and 'mysql' are supported"
+          raise ConfigError, "only 'mysql' is supported"
         end
 
         @last_time = Time.now.to_i
@@ -79,7 +85,6 @@ module PerfectQueue
 SELECT id, timeout, data, created_at, resource
 FROM `#{@table}`
 WHERE timeout <= ? AND timeout <= ? AND created_at IS NOT NULL
-  AND #{EVENT_HORIZON} < timeout
 ORDER BY timeout ASC
 LIMIT ?
 SQL
@@ -94,37 +99,11 @@ LEFT JOIN (
   GROUP BY resource
 ) AS R ON resource = res
 WHERE timeout <= ? AND created_at IS NOT NULL AND (max_running-running IS NULL OR max_running-running > 0)
-  AND #{EVENT_HORIZON} < timeout
 ORDER BY weight DESC, timeout ASC
 LIMIT ?
 SQL
         end
 
-        case url.split('//',2)[0].to_s
-        when /sqlite/i
-          # sqlite always locks tables on BEGIN
-          @table_lock = nil
-          @table_unlock = nil
-        when /mysql/i
-          @table_lock = lambda {
-            locked = nil
-            loop do
-              @db.fetch("SELECT GET_LOCK('#{@table}', #{LOCK_WAIT_TIMEOUT}) locked") do |row|
-                locked = true if row[:locked] == 1
-              end
-              break if locked
-            end
-          }
-          @table_unlock = lambda {
-            @db.run("SELECT RELEASE_LOCK('#{@table}')")
-          }
-        else
-          @table_lock = lambda {
-            @db.run("LOCK TABLE `#{@table}`")
-          }
-          @table_unlock = nil
-        end
-
         @prefetch_break_types = config[:prefetch_break_types] || []
 
         @cleanup_interval = config[:cleanup_interval] || DEFAULT_DELETE_INTERVAL
@@ -190,6 +169,24 @@ SQL
         }
       end
 
+      def compress_data(data, compression)
+        if compression == 'gzip'
+          require 'zlib'
+          require 'stringio'
+          io = StringIO.new
+          io.set_encoding(Encoding::ASCII_8BIT)
+          gz = Zlib::GzipWriter.new(io)
+          begin
+            gz.write(data)
+          ensure
+            gz.close
+          end
+          data = io.string
+          data = Sequel::SQL::Blob.new(data)
+        end
+        data
+      end
+
       # => Task
       def submit(key, type, data, options)
         now = (options[:now] || Time.now).to_i
@@ -200,23 +197,7 @@ SQL
         max_running = options[:max_running]
         data = data ? data.dup : {}
         data['type'] = type
-
-        d = data.to_json
-
-        if options[:compression] == 'gzip'
-          require 'zlib'
-          require 'stringio'
-          io = StringIO.new
-          gz = Zlib::GzipWriter.new(io)
-          begin
-            gz.write(d)
-          ensure
-            gz.close
-          end
-          d = io.string
-          d.force_encoding('ASCII-8BIT') if d.respond_to?(:force_encoding)
-          d = Sequel::SQL::Blob.new(d)
-        end
+        d = compress_data(data.to_json, options[:compression])
 
         connect {
           begin
@@ -240,7 +221,7 @@ SQL
 
         if @cleanup_interval_count <= 0
           connect_locked {
-            @db["DELETE FROM `#{@table}` WHERE #{EVENT_HORIZON} < timeout && timeout <= ? AND created_at IS NULL", now].delete
+            @db["DELETE FROM `#{@table}` WHERE timeout <= ? AND created_at IS NULL", now].delete
             @cleanup_interval_count = @cleanup_interval
           }
         end
@@ -281,8 +262,6 @@ SQL
 
       # => nil
       def cancel_request(key, options)
-        now = (options[:now] || Time.now).to_i
-
         # created_at=0 means cancel_requested
         connect {
           n = @db["UPDATE `#{@table}` SET created_at=0 WHERE id=? AND created_at IS NOT NULL", key].update
@@ -323,7 +302,7 @@ SQL
         params = [sql, next_timeout]
         if data
           sql << ", data=?"
-          params << data.to_json
+          params << compress_data(data.to_json, options[:compression])
         end
         sql << " WHERE id=? AND created_at IS NOT NULL"
         params << key
@@ -336,8 +315,6 @@ SQL
               raise PreemptedError, "task key=#{key} does not exist or preempted."
             elsif row[:created_at] == nil
               raise PreemptedError, "task key=#{key} preempted."
-            elsif row[:created_at] <= 0
-              raise CancelRequestedError, "task key=#{key} is cancel requested."
             else # row[:timeout] == next_timeout
               # ok
             end
@@ -395,6 +372,8 @@ SQL
               else
                 STDERR.puts err + "\n  abort."
               end
+            else
+              err = $!
             end
 
             STDERR.puts "disconnects current connection: #{err}"
@@ -411,6 +390,7 @@ SQL
       GZIP_MAGIC_BYTES = [0x1f, 0x8b].pack('CC')
 
       def create_attributes(now, row)
+        compression = nil
         if row[:created_at] === nil
           created_at = nil  # unknown creation time
           status = TaskStatus::FINISHED
@@ -434,6 +414,7 @@ SQL
           if d[0, 2] == GZIP_MAGIC_BYTES
             require 'zlib'
             require 'stringio'
+            compression = 'gzip'
             gz = Zlib::GzipReader.new(StringIO.new(d))
             begin
               d = gz.read
@@ -464,6 +445,7 @@ SQL
           :max_running => row[:max_running],
           :message => nil,  # not supported
           :node => nil,  # not supported
+          :compression => compression,
         }
       end
 

data/lib/perfectqueue/engine.rb

@@ -56,7 +56,7 @@ module PerfectQueue
           @processors << @processor_class.new(@runner, @processors.size+1, config)
         end
       elsif extra < 0
-        -extra.times do
+        (-extra).times do
           c = @processors.shift
           c.stop(immediate)
           c.join
@@ -106,10 +106,7 @@ module PerfectQueue
     def replace(immediate, command=[$0]+ARGV)
       return if @replaced_pid
       stop(immediate)
-      @replaced_pid = Process.fork do
-        exec(*command)
-        exit!(127)
-      end
+      @replaced_pid = Process.spawn(*command)
       self
     end
 

data/lib/perfectqueue/multiprocess/thread_processor.rb

@@ -121,7 +121,7 @@ module PerfectQueue
       end
 
       def process(task)
-        @log.info "acquired task task=#{task.key} id=#{@processor_id}: #{task.inspect}"
+        @log.info "acquired task task=#{task.key} id=#{@processor_id}"
         begin
           r = @runner.new(task)
           @tm.set_task(task, r)

data/lib/perfectqueue/task.rb

@@ -58,6 +58,7 @@ module PerfectQueue
   class TaskWithMetadata < Task
     def initialize(client, key, attributes)
       super(client, key)
+      @compression = attributes.delete(:compression)
       @attributes = attributes
     end
 
@@ -93,7 +94,7 @@ module PerfectQueue
     def update_data!(hash)
       data = @attributes[:data] || {}
       merged = data.merge(hash)
-      heartbeat!(:data => merged)
+      heartbeat!(data: merged, compression: compression)
       @attributes[:data] = merged
     end
 

data/lib/perfectqueue/task_metadata.rb

@@ -19,6 +19,7 @@
 module PerfectQueue
   module TaskMetadataAccessors
     attr_reader :attributes
+    attr_reader :compression
 
     def type
       @attributes[:type]
@@ -60,10 +61,6 @@ module PerfectQueue
       status == TaskStatus::FINISHED
     end
 
-    def running?
-      status == TaskStatus::RUNNING
-    end
-
     def waiting?
       status == TaskStatus::WAITING
     end
@@ -83,6 +80,7 @@ module PerfectQueue
     def initialize(client, key, attributes)
       super(client)
       @key = key
+      @compression = attributes.delete(:compression)
       @attributes = attributes
     end
 

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.44.1"
+  VERSION = "0.8.45"
 end

data/perfectqueue.gemspec

@@ -19,8 +19,7 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "sequel", "~> 3.48.0"
   gem.add_development_dependency "rake", "~> 0.9.2"
-  gem.add_development_dependency "rspec", "~> 2.10.0"
+  gem.add_development_dependency "rspec", "~> 3.3.0"
   gem.add_development_dependency "simplecov", "~> 0.10.0"
-  gem.add_development_dependency "sqlite3", "~> 1.3.3"
   gem.add_development_dependency "mysql2", "~> 0.3.20"
 end

data/spec/application/base_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe PerfectQueue::Application::Base do
+  describe '.decider=' do
+    it 'defines .decider which returns the decider' do
+      decider_klass = double('decider_klass')
+      klass = PerfectQueue::Application::Base
+      allow(klass).to receive(:decider).and_call_original
+      allow(klass).to receive(:decider=).with(decider_klass).and_call_original
+      expect(klass.decider = decider_klass).to eq(decider_klass)
+      expect(klass.decider).to eq(decider_klass)
+    end
+  end
+
+  describe '.decider' do
+    it 'returns DefaultDecider' do
+      expect(PerfectQueue::Application::Base.decider).to eq(PerfectQueue::Application::DefaultDecider)
+    end
+  end
+
+  describe '#new' do
+    let (:task){ double('task') }
+    let (:base) { PerfectQueue::Application::Base.new(task) }
+    it 'calls super and set decider'do
+      expect(base).to be_an_instance_of(PerfectQueue::Application::Base)
+      expect(base.instance_variable_get(:@task)).to eq(task)
+      expect(base.instance_variable_get(:@decider)).to be_an_instance_of(Application::DefaultDecider)
+    end
+  end
+
+  describe '#run' do
+    let (:base) { PerfectQueue::Application::Base.new(double('task')) }
+    it 'returns nil if before_perform returns false' do
+      allow(base).to receive(:before_perform).and_return(false)
+      expect(base.run).to be_nil
+    end
+    it 'returns nil' do
+      expect(base).to receive(:before_perform).exactly(:once).and_call_original
+      expect(base).to receive(:perform).exactly(:once).and_return(nil)
+      expect(base).to receive(:after_perform).exactly(:once).and_call_original
+      expect(base.run).to be_nil
+    end
+    it 'calls unexpected_error_raised on error' do
+      allow(base).to receive(:before_perform).exactly(:once).and_call_original
+      allow(base).to receive(:perform).exactly(:once) { raise }
+      allow(base).to receive(:decide!).with(:unexpected_error_raised, error: kind_of(Exception)).exactly(:once)
+      expect(base.run).to be_nil
+    end
+  end
+
+  describe '#before_perform' do
+    let (:base) { PerfectQueue::Application::Base.new(double('task')) }
+    it 'returns true' do
+      expect(base.before_perform).to be true
+    end
+  end
+
+  describe '#after_perform' do
+    let (:base) { PerfectQueue::Application::Base.new(double('task')) }
+    it 'returns nil' do
+      expect(base.after_perform).to be_nil
+    end
+  end
+
+  describe '#decide!' do
+    let (:base) do
+      decider = double('decider')
+      expect(decider).to receive(:decide!).with(:type, :option).exactly(:once)
+      decider_klass = double('decider_klass')
+      allow(decider_klass).to receive(:new).with(kind_of(PerfectQueue::Application::Base)).and_return(decider)
+      klass = PerfectQueue::Application::Base
+      allow(klass).to receive(:decider).and_call_original
+      allow(klass).to receive(:decider=).with(decider_klass).and_call_original
+      klass.decider = decider_klass
+      klass.new(double('task'))
+    end
+    it 'calls decider.decide' do
+      expect(base.decide!(:type, :option)).to be_nil
+    end
+  end
+end

data/spec/application/decider_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe PerfectQueue::Application::UndefinedDecisionError do
+  it { is_expected.to be_an_instance_of(PerfectQueue::Application::UndefinedDecisionError) }
+  it { is_expected.to be_a(Exception) }
+end
+
+describe PerfectQueue::Application::Decider do
+  describe '#new' do
+    let (:decider) { PerfectQueue::Application::Decider.new(nil) }
+    it do
+      expect(decider).to be_an_instance_of(PerfectQueue::Application::Decider)
+    end
+  end
+
+  describe '#queue' do
+    let (:queue){ double('queue') }
+    let (:decider) do
+      base = double('base')
+      allow(base).to receive(:queue).exactly(:once).and_return(queue)
+      PerfectQueue::Application::Decider.new(base)
+    end
+    it 'calls @base.queue' do
+      expect(decider.queue).to eq(queue)
+    end
+  end
+
+  describe '#task' do
+    let (:task){ double('task') }
+    let (:decider) do
+      base = double('base')
+      allow(base).to receive(:task).exactly(:once).and_return(task)
+      PerfectQueue::Application::Decider.new(base)
+    end
+    it 'calls @base.task' do
+      expect(decider.task).to eq(task)
+    end
+  end
+
+  describe '#decide!' do
+    let (:decider) { PerfectQueue::Application::Decider.new(nil) }
+    it 'calls the specified method' do
+      allow(decider).to receive(:foo).exactly(:once).with(72).and_return(42)
+      expect(decider.decide!(:foo, 72)).to eq(42)
+    end
+    it 'raises UndefinedDecisionError on unknown method' do
+      expect{ decider.decide!(:foo, 72) }.to raise_error(PerfectQueue::Application::UndefinedDecisionError)
+    end
+  end
+end
+
+describe PerfectQueue::Application::DefaultDecider do
+  subject { PerfectQueue::Application::DefaultDecider.new(nil) }
+  it { is_expected.to be_a(PerfectQueue::Application::Decider) }
+  it { is_expected.to be_an_instance_of(PerfectQueue::Application::DefaultDecider) }
+end