perfectqueue 0.7.7 → 0.7.8

data/ChangeLog

@@ -1,4 +1,9 @@
 
+== 2011-12-17 version 0.7.8
+
+* Supports resource limit
+
+
 == 2011-12-08 version 0.7.7
 
 * RDBBackend doesn't create table automatically

data/lib/perfectqueue/backend.rb

@@ -3,13 +3,14 @@ module PerfectQueue
 
 
 class Task
-  def initialize(id, created_at, data)
+  def initialize(id, created_at, data, resource=nil)
     @id = id
     @created_at = created_at
     @data = data
+    @resource = resource
   end
 
-  attr_reader :id, :created_at, :data
+  attr_reader :id, :created_at, :data, :resource
 end
 
 
@@ -39,7 +40,7 @@ class Backend
   end
 
   # => true (success) or nil (already exists)
-  def submit(id, data, time=Time.now.to_i)
+  def submit(id, data, time=Time.now.to_i, resource=nil)
   end
 
   def close

data/lib/perfectqueue/backend/null.rb

@@ -23,7 +23,7 @@ class NullBackend < Backend
     true
   end
 
-  def submit(id, data, time=Time.now.to_i)
+  def submit(id, data, time=Time.now.to_i, resource=nil)
     true
   end
 end

data/lib/perfectqueue/backend/rdb.rb

@@ -14,33 +14,23 @@ class RDBBackend < Backend
     }
   end
 
-  private
-  #def init_db(type)
-  #  sql = ''
-  #  case type
-  #  when /mysql/i
-  #    sql << "CREATE TABLE IF NOT EXISTS `#{@table}` ("
-  #    sql << "  id VARCHAR(256) NOT NULL,"
-  #    sql << "  timeout INT NOT NULL,"
-  #    sql << "  data BLOB NOT NULL,"
-  #    sql << "  created_at INT,"
-  #    sql << "  PRIMARY KEY (id)"
-  #    sql << ") ENGINE=INNODB;"
-  #  else
-  #    sql << "CREATE TABLE IF NOT EXISTS `#{@table}` ("
-  #    sql << "  id VARCHAR(256) NOT NULL,"
-  #    sql << "  timeout INT NOT NULL,"
-  #    sql << "  data BLOB NOT NULL,"
-  #    sql << "  created_at INT,"
-  #    sql << "  PRIMARY KEY (id)"
-  #    sql << ");"
-  #  end
-  #  # TODO index
-  #  connect {
-  #    @db.run sql
-  #  }
-  #end
+  def create_tables
+    sql = ''
+    sql << "CREATE TABLE IF NOT EXISTS `#{@table}` ("
+    sql << "  id VARCHAR(256) NOT NULL,"
+    sql << "  timeout INT NOT NULL,"
+    sql << "  data BLOB NOT NULL,"
+    sql << "  created_at INT,"
+    sql << "  resource VARCHAR(256),"
+    sql << "  PRIMARY KEY (id)"
+    sql << ");"
+    # TODO index
+    connect {
+      @db.run sql
+    }
+  end
 
+  private
   def connect(&block)
     begin
       block.call
@@ -51,35 +41,41 @@ class RDBBackend < Backend
 
   public
   def list(&block)
-    @db.fetch("SELECT id, timeout, data, created_at FROM `#{@table}` WHERE created_at IS NOT NULL ORDER BY timeout ASC;") {|row|
-      yield row[:id], row[:created_at], row[:data], row[:timeout]
+    @db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` WHERE created_at IS NOT NULL ORDER BY timeout ASC;") {|row|
+      yield row[:id], row[:created_at], row[:data], row[:timeout], row[:resource]
     }
   end
 
   MAX_SELECT_ROW = 4
+  MAX_RESOURCE = 4
 
   def acquire(timeout, now=Time.now.to_i)
     connect {
-      while true
-        rows = 0
-        @db.fetch("SELECT id, timeout, data, created_at FROM `#{@table}` WHERE timeout <= ? ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW};", now) {|row|
-
-          unless row[:created_at]
-            # finished/canceled task
-            @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
-
-          else
-            n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?;", timeout, row[:id], row[:timeout]].update
-            if n > 0
-              return row[:id], Task.new(row[:id], row[:created_at], row[:data])
+      @db.run "LOCK TABLES `#{@table}` WRITE;" rescue nil  # TODO mysql only
+      begin
+        while true
+          rows = 0
+          @db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` AS T WHERE timeout <= ? AND (resource IS NULL OR (SELECT count(1) FROM `#{@table}` WHERE timeout > ? AND resource = T.resource) < #{MAX_RESOURCE}) ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW};", now, now) {|row|
+
+            unless row[:created_at]
+              # finished/canceled task
+              @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
+
+            else
+              n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?;", timeout, row[:id], row[:timeout]].update
+              if n > 0
+                return row[:id], Task.new(row[:id], row[:created_at], row[:data], row[:resource])
+              end
             end
-          end
 
-          rows += 1
-        }
-        if rows < MAX_SELECT_ROW
-          return nil
+            rows += 1
+          }
+          if rows < MAX_SELECT_ROW
+            return nil
+          end
         end
+      ensure
+        @db.run "UNLOCK TABLES `#{@table}`;" rescue nil  # TODO mysql only
       end
     }
   end
@@ -105,10 +101,10 @@ class RDBBackend < Backend
     finish(id, delete_timeout, now)
   end
 
-  def submit(id, data, time=Time.now.to_i)
+  def submit(id, data, time=Time.now.to_i, resource=nil)
     connect {
       begin
-        n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at) VALUES (?, ?, ?, ?);", id, time, data, time].insert
+        n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at, resource) VALUES (?, ?, ?, ?, ?);", id, time, data, time, resource].insert
         return true
       rescue Sequel::DatabaseError
         return nil

data/lib/perfectqueue/backend/simpledb.rb

@@ -26,6 +26,7 @@ class SimpleDBBackend < Backend
   end
 
   def list(&block)
+    # TODO support resource limit
     @domain.items.select('timeout', 'data', 'created_at',
                         :where => "created_at != '' AND timeout > '#{int_encode(0)}'",
                         :order => [:timeout, :asc],
@@ -36,7 +37,7 @@ class SimpleDBBackend < Backend
       created_at = int_decode(attrs['created_at'].first)
       data = attrs['data'].first
       timeout = int_decode(attrs['timeout'].first)
-      yield id, created_at, data, timeout
+      yield id, created_at, data, timeout, nil
     }
   end
 
@@ -44,6 +45,7 @@ class SimpleDBBackend < Backend
 
   def acquire(timeout, now=Time.now.to_i)
     while true
+      # TODO support resource limit
       rows = 0
       @domain.items.select('timeout', 'data', 'created_at',
                           :where => "timeout <= '#{int_encode(now)}'",
@@ -66,7 +68,7 @@ class SimpleDBBackend < Backend
 
             data = attrs['data'].first
 
-            return [id,salt], Task.new(id, created_at, data)
+            return [id,salt], Task.new(id, created_at, data, nil)
           end
 
         rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
@@ -111,7 +113,8 @@ class SimpleDBBackend < Backend
     finish(token, delete_timeout, now)
   end
 
-  def submit(id, data, time=Time.now.to_i)
+  def submit(id, data, time=Time.now.to_i, resource=nil)
+    # TODO support resource limit
     begin
       @domain.items[id].attributes.replace('timeout'=>int_encode(time), 'created_at'=>int_encode(time), 'data'=>data,
           :unless=>'timeout')

data/lib/perfectqueue/command/perfectqueue.rb

@@ -270,12 +270,12 @@ backend = backend_proc.call
 
 case type
 when :list
-  format = "%26s %26s %26s  %s"
-  puts format % ["id", "created_at", "timeout", "data"]
+  format = "%26s %26s %26s %26s  %s"
+  puts format % ["id", "created_at", "timeout", "resource", "data"]
   time_format = "%Y-%m-%d %H:%M:%S %z"
   n = 0
-  backend.list {|id,created_at,data,timeout|
-    puts format % [id, Time.at(created_at).strftime(time_format), Time.at(timeout).strftime(time_format), data]
+  backend.list {|id,created_at,data,timeout,resource|
+    puts format % [id, Time.at(created_at).strftime(time_format), Time.at(timeout).strftime(time_format), resource, data]
     n += 1
   }
   puts "#{n} entries."

data/lib/perfectqueue/version.rb

@@ -1,5 +1,5 @@
 module PerfectQueue
 
-VERSION = '0.7.7'
+VERSION = '0.7.8'
 
 end

data/test/backend_test.rb

@@ -15,8 +15,10 @@ class BackendTest < Test::Unit::TestCase
   end
 
   def open_backend
-    PerfectQueue::SimpleDBBackend.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], 'perfectqueue-test-1').use_consistent_read
-    #PerfectQueue::RDBBackend.new(DB_URI, "perfectdb_test")
+    #PerfectQueue::SimpleDBBackend.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], 'perfectqueue-test-1').use_consistent_read
+    db = PerfectQueue::RDBBackend.new(DB_URI, "perfectdb_test")
+    db.create_tables
+    db
   end
 
   it 'acquire' do
@@ -213,5 +215,33 @@ class BackendTest < Test::Unit::TestCase
     assert_equal time, task.created_at
     assert_equal 'data1', task.data
   end
+
+  it 'resource' do
+    clean_backend
+
+    db1 = open_backend
+
+    time = Time.now.to_i
+
+    5.times do |i|
+      ok = db1.submit(@key_prefix+'test'+i.to_s, 'data1', time, 'user1')
+      assert_equal true, ok
+    end
+    ok = db1.submit(@key_prefix+'test5', 'data2', time, 'user2')
+    assert_equal true, ok
+
+    4.times do
+      token, task = db1.acquire(time+TIMEOUT, time)
+      assert_not_equal nil, task
+      assert_equal "user1", task.resource
+    end
+
+    token, task = db1.acquire(time+TIMEOUT, time)
+    assert_not_equal nil, task
+    assert_equal "user2", task.resource
+
+    token, task = db1.acquire(time+TIMEOUT, time)
+    assert_equal nil, task
+  end
 end
 

data/test/test_helper.rb

@@ -1,3 +1,5 @@
+require 'simplecov'
+SimpleCov.start
 require 'test/unit'
 $LOAD_PATH << File.dirname(__FILE__)+"/../lib"
 require 'perfectqueue'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.7.7
+  version: 0.7.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-09 00:00:00.000000000Z
+date: 2011-12-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70289971021580 !ruby/object:Gem::Requirement
+  requirement: &70113160282360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 3.26.0
   type: :runtime
   prerelease: false
-  version_requirements: *70289971021580
+  version_requirements: *70113160282360
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &70289971017640 !ruby/object:Gem::Requirement
+  requirement: &70113160281640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,7 +32,7 @@ dependencies:
         version: 1.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *70289971017640
+  version_requirements: *70113160281640
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -82,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.12
 signing_key: 
 specification_version: 3
 summary: Highly available distributed queue built on RDBMS or SimpleDB