perfectqueue 0.8.45 → 0.8.46

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca67d530351a8a7e0ff2f950d10248bc23eb5e2e
-  data.tar.gz: 5b66bc3219182f721af358e0bce7836ffdca356d
+  metadata.gz: e091df346ddc80d849184494788572dbbe07919e
+  data.tar.gz: 40533e8a9adf12acc69d85273675ba9043743c8b
 SHA512:
-  metadata.gz: 282b4e05105a5b6fe92cfb3c406b415221784350cb1449e87fe42b481fa7adc0b5d5318c67cc5922e9deafbd244939b2481e76c10ced8b15e981605eacddde35
-  data.tar.gz: 7f4a15e96b072fa04144d04f6f92396a730b76de07c0cb45c6f63d9c30d84bbfff88d7cab526a7a895d6265ed74cc897f9d0b8293d3a7c8271d67cba65c4acee
+  metadata.gz: 253f1165eebc079a3ad181add6e6a516bb3457000de54172470a62a8ac26335208a1bbb135df77b1a8ad28ad82103426d8321f70954f357a5b94c410b6e7e15c
+  data.tar.gz: d37142ec6e8927b4708bb8f223d61ac3a5618ebccfae7b70239612883e0a1739c69c4910c2cc4390de12756268cad248ea34cdb2300a213e6bf348b87cf3a33c

data/ChangeLog

@@ -1,3 +1,10 @@
+== 2016-03-23 version 0.8.46
+
+* Introduce new timeout model
+  NOTE: this change requires migration process like actual DELETE query
+  see WHERE timeout <= now AND created_at IS NULL.
+  80946079a91dd89c8e371e9b3fdd7f4805c7e4f9
+
 == 2015-12-02 version 0.8.45
 
 * remove sqlite3 support

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -21,6 +21,18 @@ module PerfectQueue
     class RDBCompatBackend
       include BackendHelper
 
+      #
+      # == timeout model
+      #
+      # 0 ---- now-1Bs ---- retention ---|----- now -- alive ------- FUTURE
+      #   ~~~~~~~^  to be deleted ^      |~~~^~~~       ^ running or in-queue
+      #    DELETE          13_0000_0000->|   to be acquired
+      #
+      # NOTE: this architecture introduces Year 2042 problem.
+      #
+      DELETE_OFFSET = 10_0000_0000
+      EVENT_HORIZON = 13_0000_0000 # 2011-03-13 07:06:40 UTC
+
       class Token < Struct.new(:key)
       end
 
@@ -34,24 +46,8 @@ module PerfectQueue
           raise ConfigError, ":table option is required"
         end
 
-        #password = config[:password]
-        #user = config[:user]
-
-        if /\Amysql/i =~ url
-          require 'uri'
-
-          uri = URI.parse(url)
-          options = {
-            user: uri.user,
-            password: uri.password,
-            host: uri.host,
-            port: uri.port ? uri.port.to_i : 3306,
-            max_connections: 1
-          }
-          options[:sslca] = config[:sslca] if config[:sslca]
-
-          db_name = uri.path.split('/')[1]
-          @db = Sequel.mysql2(db_name, options)
+        if /\Amysql2:/i =~ url
+          @db = Sequel.connect(url, {max_connections: 1, sslca: config[:sslca]})
           if config.fetch(:use_connection_pooling, nil) != nil
             @use_connection_pooling = !!config[:use_connection_pooling]
           else
@@ -67,7 +63,7 @@ module PerfectQueue
             end
           }
           @table_unlock = lambda {
-            @db.run("SELECT RELEASE_LOCK('#{@table}')")
+            @db.run("DO RELEASE_LOCK('#{@table}')")
           }
         else
           raise ConfigError, "only 'mysql' is supported"
@@ -84,7 +80,8 @@ module PerfectQueue
           @sql = <<SQL
 SELECT id, timeout, data, created_at, resource
 FROM `#{@table}`
-WHERE timeout <= ? AND timeout <= ? AND created_at IS NOT NULL
+WHERE #{EVENT_HORIZON} < timeout AND timeout <= ? AND timeout <= ?
+      AND created_at IS NOT NULL
 ORDER BY timeout ASC
 LIMIT ?
 SQL
@@ -98,7 +95,9 @@ LEFT JOIN (
   WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
   GROUP BY resource
 ) AS R ON resource = res
-WHERE timeout <= ? AND created_at IS NOT NULL AND (max_running-running IS NULL OR max_running-running > 0)
+WHERE #{EVENT_HORIZON} < timeout AND timeout <= ?
+      AND created_at IS NOT NULL
+      AND (max_running-running IS NULL OR max_running-running > 0)
 ORDER BY weight DESC, timeout ASC
 LIMIT ?
 SQL
@@ -220,8 +219,9 @@ SQL
         tasks = []
 
         if @cleanup_interval_count <= 0
-          connect_locked {
-            @db["DELETE FROM `#{@table}` WHERE timeout <= ? AND created_at IS NULL", now].delete
+          delete_timeout = now - DELETE_OFFSET
+          connect { # TODO: HERE should be still connect_locked ?
+            @db["DELETE FROM `#{@table}` WHERE timeout <= ? AND created_at IS NULL", delete_timeout].delete
             @cleanup_interval_count = @cleanup_interval
           }
         end
@@ -243,15 +243,16 @@ SQL
             return nil
           end
 
-          sql = "UPDATE `#{@table}` SET timeout=? WHERE id IN ("
-          params = [sql, next_timeout]
+          sql = "UPDATE `#{@table}` SET timeout=? WHERE timeout <= ? AND id IN ("
+          params = [sql, next_timeout, now]
           tasks.each {|t| params << t.key }
           sql << (1..tasks.size).map { '?' }.join(',')
           sql << ") AND created_at IS NOT NULL"
 
           n = @db[*params].update
           if n != tasks.size
-            # TODO table lock doesn't work. error?
+            # NOTE table lock doesn't work. error!
+            return nil
           end
 
           @cleanup_interval_count -= 1
@@ -279,7 +280,7 @@ SQL
       # => nil
       def finish(task_token, retention_time, options)
         now = (options[:now] || Time.now).to_i
-        delete_timeout = now + retention_time
+        delete_timeout = now - DELETE_OFFSET + retention_time
         key = task_token.key
 
         connect {
@@ -333,15 +334,12 @@ SQL
           locked = false
 
           begin
-            @db.transaction do
-              if @table_lock
-                @table_lock.call
-                locked = true
-              end
-
-              return block.call
+            if @table_lock
+              @table_lock.call
+              locked = true
             end
 
+            return block.call
           ensure
             if @use_connection_pooling && locked
               @table_unlock.call

data/lib/perfectqueue/multiprocess/thread_processor.rb

@@ -121,7 +121,7 @@ module PerfectQueue
       end
 
       def process(task)
-        @log.info "acquired task task=#{task.key} id=#{@processor_id}"
+        @log.info "acquired task task=#{task.key} id=#{@processor_id}: #{task.inspect}"
         begin
           r = @runner.new(task)
           @tm.set_task(task, r)

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.45"
+  VERSION = "0.8.46"
 end

data/spec/rdb_compat_backend_spec.rb

@@ -76,7 +76,7 @@ describe Backend::RDBCompatBackend do
   let (:now){ Time.now.to_i }
   let (:client){ double('client') }
   let (:table){ 'test_queues' }
-  let (:config){ {url: 'mysql://root:@localhost/perfectqueue_test', table: table} }
+  let (:config){ {url: 'mysql2://root:@localhost/perfectqueue_test', table: table} }
   let (:db) do
     d = Backend::RDBCompatBackend.new(client, config)
     s = d.db
@@ -103,12 +103,12 @@ describe Backend::RDBCompatBackend do
       expect{Backend::RDBCompatBackend.new(client, config)}.to raise_error(ConfigError)
     end
     it 'with use_connection_pooling' do
-      config = {url: 'mysql://root:@localhost/perfectqueue_test', table: table, use_connection_pooling: true}
+      config = {url: 'mysql2://root:@localhost/perfectqueue_test', table: table, use_connection_pooling: true}
       db = Backend::RDBCompatBackend.new(client, config)
       expect(db.instance_variable_get(:@use_connection_pooling)).to eq true
     end
     it 'disable_resource_limit' do
-      config = {url: 'mysql://root:@localhost/perfectqueue_test', table: table, disable_resource_limit: true}
+      config = {url: 'mysql2://root:@localhost/perfectqueue_test', table: table, disable_resource_limit: true}
       db = Backend::RDBCompatBackend.new(client, config)
       expect(db.instance_variable_get(:@sql)).not_to include('max_running')
     end
@@ -228,6 +228,26 @@ describe Backend::RDBCompatBackend do
         expect(ary[1]).to be_an_instance_of(AcquiredTask)
       end
     end
+    context 'stole a task' do
+      let :t0 do now - 100 end
+      before do
+        db.submit('key1', 'test1', nil, {now: t0})
+        db.submit('key2', 'test2', nil, {now: t0})
+        db.submit('key3', 'test3', nil, {now: t0})
+      end
+      it 'returns nil' do
+        # hook and steal a task
+        mock = double('prefetch_break_types')
+        db.instance_variable_set(:@prefetch_break_types, mock)
+        allow(mock).to receive(:include?) do
+          db.db['UPDATE `test_queues` SET timeout=? WHERE id=?', now+300, 'key2'].update
+          false
+        end
+
+        ary = db.acquire(alive_time, max_acquire, {})
+        expect(ary).to be_nil
+      end
+    end
   end
 
   context '#cancel_request' do
@@ -275,7 +295,7 @@ describe Backend::RDBCompatBackend do
     let (:key){ 'key' }
     let (:task_token){ Backend::RDBCompatBackend::Token.new(key) }
     let (:retention_time) { 42 }
-    let (:delete_timeout){ now + retention_time }
+    let (:delete_timeout){ now - Backend::RDBCompatBackend::DELETE_OFFSET + retention_time }
     let (:options){ {now: now} }
     context 'have the task' do
       before do
@@ -442,7 +462,7 @@ describe Backend::RDBCompatBackend do
     end
     it 'ensures to unlock on error with use_connection_pooling' do
       #expect(STDERR).to receive(:puts)
-      config = {url: 'mysql://root:@localhost/perfectqueue_test', table: table, use_connection_pooling: true}
+      config = {url: 'mysql2://root:@localhost/perfectqueue_test', table: table, use_connection_pooling: true}
       db1 = Backend::RDBCompatBackend.new(client, config)
       #expect{ db.__send__(:connect_locked){ raise } }.to raise_error(RuntimeError)
       db1.__send__(:connect_locked){ ret }

data/spec/rdb_stress.rb

@@ -0,0 +1,79 @@
+require 'spec_helper'
+require 'perfectqueue/backend/rdb_compat'
+require 'logger'
+
+# run this with `bundle exec rake spec SPEC_OPTS="-fd" SPEC=spec/rdb_stress.rb`
+
+describe Backend::RDBCompatBackend do
+  let (:now){ Time.now.to_i }
+  let (:client){ double('client') }
+  let (:table){ 'test_queues' }
+  let (:config){ {
+    url: 'mysql2://root:@localhost/perfectqueue_test',
+    table: table,
+    disable_resource_limit: true,
+  } }
+  let (:db) do
+    d = Backend::RDBCompatBackend.new(client, config)
+    s = d.db
+    s.tables.each{|t| s.drop_table(t) }
+    d.init_database({})
+    d
+  end
+
+  context '#acquire' do
+    let (:task_token){ Backend::RDBCompatBackend::Token.new(key) }
+    let (:alive_time){ 42 }
+    let (:max_acquire){ 10 }
+
+    context 'some tasks' do
+      before do
+        sql = nil
+        bucket_size = 200000
+        600_000.times do |i|
+          if i % bucket_size == 0
+            sql = 'INSERT `test_queues` (id, timeout, data, created_at, resource) VALUES'
+          end
+          t = now - 600 + i/1000
+          sql << "(UUID(),#{t},TO_BASE64(RANDOM_BYTES(540)),#{t},NULL),"
+          if i % bucket_size == bucket_size - 1
+            db.db.run sql.chop!
+          end
+        end
+        db.db.loggers << Logger.new($stderr)
+        db.db.sql_log_level = :debug
+      end
+      it 'returns a task' do
+        #db.instance_variable_set(:@cleanup_interval_count, 0)
+        #expect(db.db.instance_variable_get(:@default_dataset)).to receive(:delete).and_call_original
+        ary = db.acquire(alive_time, max_acquire, {})
+        expect(ary).to be_an_instance_of(Array)
+        expect(ary.size).to eq(10)
+        expect(ary[0]).to be_an_instance_of(AcquiredTask)
+      end
+    end
+
+    context 'very large jobs' do
+      before do
+        sql = nil
+        sql = 'INSERT `test_queues` (id, timeout, data, created_at, resource) VALUES'
+        data = %<UNCOMPRESS(UNCOMPRESS(FROM_BASE64('6B8AAHic7c6xCYNQFEDRh8HGAawjmAmEOE7WsPziGoJ1xnAJLbJCVgiJbpBOkHOqW96IFN34nvvYpOvyuZXPIgAAAICTS6/hku1RfR9tffQNAAAA8Icxb+7r9AO74A1h')))>
+        200.times do |i|
+          t = now - 600 + i/1000
+          sql << "(UUID(),#{t},#{data},#{t},NULL),"
+        end
+        db.db.run sql.chop!
+        db.db.loggers << Logger.new($stderr)
+        db.db.sql_log_level = :debug
+      end
+      it 'returns a task' do
+        #db.instance_variable_set(:@cleanup_interval_count, 0)
+        #expect(db.db.instance_variable_get(:@default_dataset)).to receive(:delete).and_call_original
+        ary = db.acquire(alive_time, max_acquire, {})
+        expect(ary).to be_an_instance_of(Array)
+        expect(ary.size).to eq(10)
+        expect(ary[0]).to be_an_instance_of(AcquiredTask)
+      end
+    end
+  end
+end

data/spec/task_monitor_spec.rb

@@ -40,3 +40,33 @@ describe PerfectQueue::TaskMonitor do
     end
   end
 end
+
+describe PerfectQueue::TaskMonitorHook do
+  let (:task_monitor) do
+    tm = PerfectQueue::TaskMonitor.new(logger: double('logger').as_null_object)
+  end
+  let (:task) do
+    obj = double('task', key: 'foo', finish!: 1, release!: 1, retry!: 1, cancel_request!: 1, update_data!: 1)
+    obj.extend(TaskMonitorHook)
+    obj.instance_variable_set(:@log, double('log', info: nil))
+    obj.instance_variable_set(:@task_monitor, task_monitor)
+    obj
+  end
+  before do
+  end
+  describe 'finish!' do
+    it { task.finish! }
+  end
+  describe 'release!' do
+    it { task.release! }
+  end
+  describe 'retry!' do
+    it { task.retry! }
+  end
+  describe 'cancel_request!' do
+    it { task.cancel_request! }
+  end
+  describe 'update_data!' do
+    it { task.update_data!(double) }
+  end
+end

data/spec/task_spec.rb

@@ -42,7 +42,7 @@ describe PerfectQueue::Task do
 
   describe '#update_data!' do
     context 'PLT-4238' do
-      let (:config){ {type: 'rdb_compat', url: 'mysql://root:@localhost/perfectqueue_test', table: 'test_queues', type: 'rdb_compat'} }
+      let (:config){ {type: 'rdb_compat', url: 'mysql2://root:@localhost/perfectqueue_test', table: 'test_queues'} }
       let (:client){ Client.new(config) }
       before do
         client.backend.db.tap{|s| s.tables.each{|t| s.drop_table(t) } }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.45
+  version: 0.8.46
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-02 00:00:00.000000000 Z
+date: 2016-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -144,6 +144,7 @@ files:
 - spec/multiprocess/thread_processor_spec.rb
 - spec/queue_spec.rb
 - spec/rdb_compat_backend_spec.rb
+- spec/rdb_stress.rb
 - spec/runner_spec.rb
 - spec/signal_thread_spec.rb
 - spec/spec_helper.rb
@@ -192,6 +193,7 @@ test_files:
 - spec/multiprocess/thread_processor_spec.rb
 - spec/queue_spec.rb
 - spec/rdb_compat_backend_spec.rb
+- spec/rdb_stress.rb
 - spec/runner_spec.rb
 - spec/signal_thread_spec.rb
 - spec/spec_helper.rb