perfectqueue 0.8.43 → 0.8.44

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ac7d746168b57f93e874c437b9daeeb304091f9
-  data.tar.gz: 2a2685af20791bad69f41301edc418c9bd1b0ee8
+  metadata.gz: a6f5457263e6492b228864dbc72be480b28ea20f
+  data.tar.gz: d83d97c23703a464f88db7343615dd856cd925ca
 SHA512:
-  metadata.gz: c680e72695794d25e0ccc5ef89c10e9ebe6b75ed576e639e23d9145dc1ad9d43baf8f4005c51c2b478bb03efb9db1fbdaf22332bf22238440a68be32d5ac59fd
-  data.tar.gz: fe80f8cbae924c1c832bb158c32a9ec66708793f17077075e5ba17f6466293a96c467a27748df2ad537e3b08fd2cc41b797738e453a42480030dc0ce51b99310
+  metadata.gz: d9cdd377afec8f82f13162b92073864e8d2deac00cf705a51409f3fe1343ab5a5d6716d2ed4a11daa50cd76c15bb4a9fbdbdba565003fe7673ead6fc979d2043
+  data.tar.gz: fbf7ef421de36a66fa9a1712f3fa0dcb59d9b7d648a77dd052bbb6f8c0006549165c0a8f04be4e3894dbb78c15badba37eb381a8bcf5822e9ae3bfabd948b164

data/.travis.yml

@@ -6,3 +6,11 @@ rvm:
   - ruby-head
 
 script: "bundle exec rake spec"
+
+before_script:
+  - mysql -e 'create database perfectqueue_test;'
+
+sudo: false
+
+notifications:
+  webhooks: http://td-beda.herokuapp.com/travisci_callback

data/ChangeLog

@@ -1,3 +1,8 @@
+== 2015-10-07 version 0.8.44
+
+* rdb_compat: Fix fair scheduling.
+* rdb_compat: Use `GET_LOCK` instead of `LOCK TABLES` in MySQL.
+
 == 2015-06-30 version 0.8.43
 
 * child_process: stop after running

data/Gemfile

@@ -1,3 +1,5 @@
 source "https://rubygems.org"
 
 gemspec
+
+gem 'coveralls', require: false

data/README.md

@@ -1,6 +1,7 @@
 # PerfectQueue
 
 [![Build Status](https://travis-ci.org/treasure-data/perfectqueue.svg?branch=master)](https://travis-ci.org/treasure-data/perfectqueue)
+[![Coverage Status](https://coveralls.io/repos/treasure-data/perfectqueue/badge.svg?branch=master&service=github)](https://coveralls.io/github/treasure-data/perfectqueue?branch=master)
 
 PerfectQueue is a highly available distributed queue built on top of RDBMS.
 PerfectQueue provides similar API to Amazon SQS, while PerfectQueue focues on reliability and flexible scheduling rather than scalability.
@@ -246,3 +247,13 @@ options for run:
 
     $ perfectqueue run -I. -Ilib -rconfig/boot.rb -rapps/workers/task_dispatch.rb TaskDispatch
 
+## Test
+
+Running spec utilize 'mysql2://root:@localhost/perfectqueue_test' as the connection string.
+Please install MySQL server at localhost then run;
+
+    $ mysql -h localhost -u root -e 'create database perfectqueue_test;'
+
+You can run spec.
+
+    $ bundle exec rake spec

data/circle.yml

@@ -0,0 +1,7 @@
+machine:
+  ruby:
+    version: 2.2.2
+
+database:
+  pre:
+    - mysql -e 'create database perfectqueue_test;'

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -81,7 +81,7 @@ LIMIT ?
 SQL
         else
           @sql = <<SQL
-SELECT id, timeout, data, created_at, resource, max_running, max_running/running AS weight
+SELECT id, timeout, data, created_at, resource, max_running, IFNULL(max_running, 1) / (IFNULL(running, 0) + 1) AS weight
 FROM `#{@table}`
 LEFT JOIN (
   SELECT resource AS res, COUNT(1) AS running
@@ -90,7 +90,7 @@ LEFT JOIN (
   GROUP BY resource
 ) AS R ON resource = res
 WHERE timeout <= ? AND created_at IS NOT NULL AND (max_running-running IS NULL OR max_running-running > 0)
-ORDER BY weight IS NOT NULL, weight DESC, timeout ASC
+ORDER BY weight DESC, timeout ASC
 LIMIT ?
 SQL
         end
@@ -101,14 +101,22 @@ SQL
           @table_lock = nil
           @table_unlock = nil
         when /mysql/i
-          if config[:disable_resource_limit]
-            @table_lock = "LOCK TABLES `#{@table}` WRITE"
-          else
-            @table_lock = "LOCK TABLES `#{@table}` WRITE, `#{@table}` AS T WRITE"
-          end
-          @table_unlock = "UNLOCK TABLES"
+          @table_lock = lambda {
+            locked = nil
+            loop do
+              @db.fetch("SELECT GET_LOCK('#{@table}', #{LOCK_WAIT_TIMEOUT}) locked") do |row|
+                locked = true if row[:locked] == 1
+              end
+              break if locked
+            end
+          }
+          @table_unlock = lambda {
+            @db.run("SELECT RELEASE_LOCK('#{@table}')")
+          }
         else
-          @table_lock = "LOCK TABLE `#{@table}`"
+          @table_lock = lambda {
+            @db.run("LOCK TABLE `#{@table}`")
+          }
           @table_unlock = nil
         end
 
@@ -122,21 +130,26 @@ SQL
 
       KEEPALIVE = 10
       MAX_RETRY = 10
+      LOCK_WAIT_TIMEOUT = 60
       DEFAULT_DELETE_INTERVAL = 20
 
       def init_database(options)
-        sql = %[
-            CREATE TABLE IF NOT EXISTS `#{@table}` (
-              id VARCHAR(256) NOT NULL,
-              timeout INT NOT NULL,
-              data BLOB NOT NULL,
-              created_at INT,
-              resource VARCHAR(256),
-              max_running INT,
-              PRIMARY KEY (id)
-            );]
+        sql = []
+        sql << "DROP TABLE IF EXISTS `#{@table}`" if options[:force]
+        sql << <<-SQL
+          CREATE TABLE IF NOT EXISTS `#{@table}` (
+            id VARCHAR(255) NOT NULL,
+            timeout INT NOT NULL,
+            data LONGBLOB NOT NULL,
+            created_at INT,
+            resource VARCHAR(255),
+            max_running INT,
+            PRIMARY KEY (id)
+          )
+          SQL
+        sql << "CREATE INDEX `index_#{@table}_on_timeout` ON `#{@table}` (`timeout`)"
         connect {
-          @db.run sql
+          sql.each(&@db.method(:run))
         }
       end
 
@@ -164,8 +177,7 @@ SQL
         now = (options[:now] || Time.now).to_i
 
         connect {
-          #@db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` WHERE !(created_at IS NULL AND timeout <= ?) ORDER BY timeout ASC;", now) {|row|
-          @db.fetch("SELECT id, timeout, data, created_at, resource, max_running FROM `#{@table}` ORDER BY timeout ASC", now) {|row|
+          @db.fetch("SELECT id, timeout, data, created_at, resource, max_running FROM `#{@table}` ORDER BY timeout ASC") {|row|
             attributes = create_attributes(now, row)
             task = TaskWithMetadata.new(@client, row[:id], attributes)
             yield task
@@ -203,7 +215,7 @@ SQL
 
         connect {
           begin
-            n = @db[
+            @db[
               "INSERT INTO `#{@table}` (id, timeout, data, created_at, resource, max_running) VALUES (?, ?, ?, ?, ?, ?)",
               key, run_at, d, now, user, max_running
             ].insert
@@ -341,7 +353,7 @@ SQL
           begin
             @db.transaction do
               if @table_lock
-                @db[@table_lock].update
+                @table_lock.call
                 locked = true
               end
 
@@ -350,7 +362,7 @@ SQL
 
           ensure
             if @use_connection_pooling && locked
-              @db[@table_unlock].update
+              @table_unlock.call
             end
           end
         }
@@ -437,7 +449,7 @@ SQL
           type = row[:id].split(/\./, 2)[0]
         end
 
-        attributes = {
+        {
           :status => status,
           :created_at => created_at,
           :data => data,

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.43"
+  VERSION = "0.8.44"
 end

data/perfectqueue.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "sequel", "~> 3.48.0"
   gem.add_development_dependency "rake", "~> 0.9.2"
   gem.add_development_dependency "rspec", "~> 2.10.0"
-  gem.add_development_dependency "simplecov", "~> 0.5.4"
+  gem.add_development_dependency "simplecov", "~> 0.10.0"
   gem.add_development_dependency "sqlite3", "~> 1.3.3"
+  gem.add_development_dependency "mysql2", "~> 0.3.20"
 end

data/spec/rdb_compat_backend_spec.rb

@@ -2,12 +2,7 @@ require 'spec_helper'
 require 'perfectqueue/backend/rdb_compat'
 
 describe Backend::RDBCompatBackend do
-  let :queue do
-    FileUtils.rm_f 'spec/test.db'
-    queue = PerfectQueue.open({:type=>'rdb_compat', :url=>'sqlite://spec/test.db', :table=>'test_tasks', :processor_type=>'thread'})
-    queue.client.init_database
-    queue
-  end
+  include QueueTest
 
   let :client do
     queue.client

data/spec/spec_helper.rb

@@ -11,19 +11,20 @@ if ENV['SIMPLE_COV']
   end
 end
 
+if ENV["CI"]
+  require 'coveralls'
+  Coveralls.wear!
+end
+
 require 'fileutils'
 
 module QueueTest
   def self.included(mod)
     mod.module_eval do
-      let :database_path do
-        'spec/test.db'
-      end
-
       let :queue_config do
         {
           :type => 'rdb_compat',
-          :url => "sqlite://#{database_path}",
+          :url => "mysql2://root:@localhost/perfectqueue_test",
           :table => 'test_tasks',
           :processor_type => 'thread',
           :cleanup_interval => 0,  # for test
@@ -36,8 +37,7 @@ module QueueTest
       end
 
       before do
-        FileUtils.rm_f database_path
-        queue.client.init_database
+        queue.client.init_database(:force => true)
       end
 
       after do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.43
+  version: 0.8.44
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-30 00:00:00.000000000 Z
+date: 2015-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.3.3
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.20
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.20
 description: Highly available distributed cron built on RDBMS
 email: frsyuki@gmail.com
 executables:
@@ -96,6 +110,7 @@ files:
 - README.md
 - Rakefile
 - bin/perfectqueue
+- circle.yml
 - lib/perfectqueue.rb
 - lib/perfectqueue/application.rb
 - lib/perfectqueue/application/base.rb
@@ -153,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Highly available distributed cron built on RDBMS