perfectqueue 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cec4a6907599be03eb95b756075c114c1a282218
-  data.tar.gz: 3cc969e794964142212690c24eef4aab1c45e847
+  metadata.gz: 7274d143afaecb7d6ee09cac6d4e287813dcf1bc
+  data.tar.gz: 25c4271e2bbcb2734c92234a73c59121e383834c
 SHA512:
-  metadata.gz: 99bbd684d883988af719392e15e37446eec1b7404d1c3a661875156fc4137ab2e6ddf090e706a0c13449a0fc17fcdba1e2d19d54fc3d8db17e018081f3cb4326
-  data.tar.gz: 8f2be0558cf06f6caf6664c7cf77aaddabb3470b632495bfdd0a8790cac72592072e58fd17051fb7a04459c8f5b369a77343f7d50135b8402e5c7ba40c52995d
+  metadata.gz: cc0acb6de0b1771606a5a8047f48d6c6e904722af0416291951cb87cfca8a65bd16b501099a1540e0f55ab7f9a41c650321b6cc87add520ead66b69acf61def2
+  data.tar.gz: 1b85520494ad5059f0dc5dc187eeab5810a51bd707e6931b21e1663c0425b0fcf05921ebb424f7aa1102e744a3f1e317cda7d0fe55d3fd36e947f895ba0420e3

data/ChangeLog

@@ -1,3 +1,7 @@
+== 2016-10-07 version 0.9.2
+
+* Use v08's strategy if resource limit is used (#55)
+
 == 2016-09-23 version 0.9.1
 
 * FOR UPDATE is not required while with GET_LOCK() (#54)

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -78,55 +78,7 @@ module PerfectQueue
           # connection test
         }
 
-        # MySQL's CONNECTION_ID() is a 64bit unsigned integer from the
-        # server's internal thread ID counter. It is unique while the MySQL
-        # server is running.
-        # https://bugs.mysql.com/bug.php?id=19806
-        #
-        # An acquired task is marked with next_timeout and CONNECTION_ID().
-        # Therefore while alive_time is not changed and we don't restart
-        # the server in 1 second, they won't conflict.
-        if config[:disable_resource_limit]
-          @update_sql = <<SQL
-UPDATE `#{@table}`
-  JOIN (
-  SELECT id
-    FROM `#{@table}` FORCE INDEX (`index_#{@table}_on_timeout`)
-   WHERE #{EVENT_HORIZON} < timeout AND timeout <= :now
-   ORDER BY timeout ASC
-      LIMIT :max_acquire) AS t1 USING(id)
-   SET timeout=:next_timeout, owner=CONNECTION_ID()
-SQL
-          @sql = <<SQL
-SELECT id, timeout, data, created_at, resource
-  FROM `#{@table}`
- WHERE timeout = ? AND owner = CONNECTION_ID()
-SQL
-        else
-          @update_sql = <<SQL
-UPDATE `#{@table}`
-  JOIN (
-    SELECT id, IFNULL(max_running, 1) / (IFNULL(running, 0) + 1) AS weight
-    FROM `#{@table}`
-    LEFT JOIN (
-      SELECT resource, COUNT(1) AS running
-      FROM `#{@table}` AS t1
-      WHERE timeout > :now AND resource IS NOT NULL
-      GROUP BY resource
-    ) AS t2 USING(resource)
-    WHERE #{EVENT_HORIZON} < timeout AND timeout <= :now AND IFNULL(max_running - running, 1) > 0
-    ORDER BY weight DESC, timeout ASC
-    LIMIT :max_acquire
-  ) AS t3 USING (id)
-SET timeout = :next_timeout, owner = CONNECTION_ID()
-SQL
-          @sql = <<SQL
-SELECT id, timeout, data, created_at, resource, max_running
-  FROM `#{@table}`
- WHERE timeout = ? AND owner = CONNECTION_ID()
-SQL
-        end
-
+        @disable_resource_limit = config[:disable_resource_limit]
         @cleanup_interval = config[:cleanup_interval] || DEFAULT_DELETE_INTERVAL
         # If cleanup_interval > max_request_per_child / max_acquire,
         # some processes won't run DELETE query.
@@ -240,7 +192,6 @@ SQL
       def acquire(alive_time, max_acquire, options)
         now = (options[:now] || Time.now).to_i
         next_timeout = now + alive_time
-        tasks = nil
         t0 = nil
 
         if @cleanup_interval_count <= 0
@@ -252,27 +203,11 @@ SQL
           }
         end
 
-        connect_locked {
-          t0=Process.clock_gettime(Process::CLOCK_MONOTONIC)
-          n = @db[@update_sql, next_timeout: next_timeout, now: now, max_acquire: max_acquire].update
-          if n <= 0
-            return nil
-          end
-          @table_unlock.call
-
-          tasks = []
-          @db.fetch(@sql, next_timeout) {|row|
-            attributes = create_attributes(nil, row)
-            task_token = Token.new(row[:id])
-            task = AcquiredTask.new(@client, row[:id], attributes, task_token)
-            tasks.push task
-          }
-          @cleanup_interval_count -= 1
-
-          return tasks
-        }
-      ensure
-        STDERR.puts "PQ:acquire from #{@table}:%6f sec (%d tasks)" % [Process.clock_gettime(Process::CLOCK_MONOTONIC)-t0,tasks.size] if tasks
+        if @disable_resource_limit
+          return acquire_without_resource(next_timeout, now, max_acquire)
+        else
+          return acquire_with_resource(next_timeout, now, max_acquire)
+        end
       end
 
       def force_finish(key, retention_time, options)
@@ -454,7 +389,98 @@ SQL
         }
       end
 
+      def acquire_without_resource(next_timeout, now, max_acquire)
+        # MySQL's CONNECTION_ID() is a 64bit unsigned integer from the
+        # server's internal thread ID counter. It is unique while the MySQL
+        # server is running.
+        # https://bugs.mysql.com/bug.php?id=19806
+        #
+        # An acquired task is marked with next_timeout and CONNECTION_ID().
+        # Therefore while alive_time is not changed and we don't restart
+        # the server in 1 second, they won't conflict.
+        update_sql = <<SQL
+UPDATE `#{@table}`
+  JOIN (
+  SELECT id
+    FROM `#{@table}` FORCE INDEX (`index_#{@table}_on_timeout`)
+   WHERE #{EVENT_HORIZON} < timeout AND timeout <= :now
+   ORDER BY timeout ASC
+      LIMIT :max_acquire) AS t1 USING(id)
+   SET timeout=:next_timeout, owner=CONNECTION_ID()
+SQL
+        select_sql = <<SQL
+SELECT id, timeout, data, created_at, resource
+  FROM `#{@table}`
+ WHERE timeout = ? AND owner = CONNECTION_ID()
+SQL
+        t0 = 0
+        connect_locked do
+          t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          n = @db[update_sql, next_timeout: next_timeout, now: now, max_acquire: max_acquire].update
+          @table_unlock.call
+          STDERR.puts "PQ:acquire from #{@table}:%6f sec (%d tasks)" % [Process.clock_gettime(Process::CLOCK_MONOTONIC)-t0,n]
+          return nil if n <= 0
+
+          tasks = []
+          @db.fetch(select_sql, next_timeout) do |row|
+            attributes = create_attributes(nil, row)
+            task_token = Token.new(row[:id])
+            task = AcquiredTask.new(@client, row[:id], attributes, task_token)
+            tasks.push task
+          end
+          @cleanup_interval_count -= 1
+          return tasks
+        end
+      end
+
+      def acquire_with_resource(next_timeout, now, max_acquire)
+        t0 = nil
+        tasks = nil
+        sql = <<SQL
+SELECT id, timeout, data, created_at, resource, max_running, IFNULL(max_running, 1) / (IFNULL(running, 0) + 1) AS weight
+FROM `#{@table}`
+LEFT JOIN (
+  SELECT resource AS res, COUNT(1) AS running
+  FROM `#{@table}` AS T
+  WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
+  GROUP BY resource
+) AS R ON resource = res
+WHERE #{EVENT_HORIZON} < timeout AND timeout <= ?
+      AND created_at IS NOT NULL
+      AND (max_running-running IS NULL OR max_running-running > 0)
+ORDER BY weight DESC, timeout ASC
+LIMIT ?
+SQL
+        connect_locked do
+          t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          tasks = []
+          @db.fetch(sql, now, now, max_acquire) do |row|
+            attributes = create_attributes(nil, row)
+            task_token = Token.new(row[:id])
+            task = AcquiredTask.new(@client, row[:id], attributes, task_token)
+            tasks.push task
+          end
+          return nil if tasks.empty?
+
+          sql = "UPDATE `#{@table}` FORCE INDEX (PRIMARY) SET timeout=? WHERE timeout <= ? AND id IN ("
+          params = [sql, next_timeout, now]
+          params.concat tasks.map(&:key)
+          sql << '?,' * tasks.size
+          sql.chop!
+          sql << ") AND created_at IS NOT NULL"
+
+          n = @db[*params].update
+          if n != tasks.size
+            # preempted
+            return nil
+          end
+        end
+        @cleanup_interval_count -= 1
+        return tasks
+      ensure
+        STDERR.puts "PQ:acquire from #{@table}:%6f sec (%d tasks)" % \
+          [Process.clock_gettime(Process::CLOCK_MONOTONIC)-t0, tasks.size] if tasks
+      end
     end
   end
 end
-

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.9.1"
+  VERSION = "0.9.2"
 end

data/spec/rdb_compat_backend_spec.rb

@@ -96,7 +96,7 @@ describe Backend::RDBCompatBackend do
     end
     it 'supports mysql' do
       expect(Backend::RDBCompatBackend.new(client, config)).to be_an_instance_of(Backend::RDBCompatBackend)
-      expect(db.instance_variable_get(:@sql)).to include('max_running')
+      expect(db.instance_variable_get(:@disable_resource_limit)).to be_falsey
     end
     it 'doesn\'t support postgres' do
       config = {url: 'postgres://localhost', table: table}
@@ -110,7 +110,7 @@ describe Backend::RDBCompatBackend do
     it 'disable_resource_limit' do
       config = {url: 'mysql2://root:@localhost/perfectqueue_test', table: table, disable_resource_limit: true}
       db = Backend::RDBCompatBackend.new(client, config)
-      expect(db.instance_variable_get(:@sql)).not_to include('max_running')
+      expect(db.instance_variable_get(:@disable_resource_limit)).to be_truthy
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-23 00:00:00.000000000 Z
+date: 2016-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel