perfectqueue 0.7.32 → 0.8.0

data/lib/perfectqueue/backend/null.rb

@@ -1,33 +0,0 @@
-
-module PerfectQueue
-
-
-class NullBackend < Backend
-  def list(&block)
-    nil
-  end
-
-  def acquire(timeout, now=Time.now.to_i)
-    nil
-  end
-
-  def finish(token, delete_timeout=3600, now=Time.now.to_i)
-    true
-  end
-
-  def update(token, timeout)
-    nil
-  end
-
-  def cancel(id, delete_timeout=3600, now=Time.now.to_i)
-    true
-  end
-
-  def submit(id, data, time=Time.now.to_i, resource=nil, max_running=nil)
-    true
-  end
-end
-
-
-end
-

data/lib/perfectqueue/backend/rdb.rb

@@ -1,181 +0,0 @@
-
-module PerfectQueue
-
-
-class RDBBackend < Backend
-  DELETE_OFFSET = 10_0000_0000
-
-  def initialize(uri, table, config={})
-    require 'sequel'
-    require 'uri'
-
-    @uri = uri
-    @table = table
-
-    u = URI.parse(@uri)
-    options = {
-      max_connections: 1,
-      user: u.user,
-      password: u.password,
-      host: u.host,
-      port: u.port ? u.port.to_i : 3306
-    }
-    options[:sslca] = config[:sslca] if config[:sslca]
-    db_name = u.path.split('/')[1]
-    @db = Sequel.mysql2(db_name, options)
-
-    #@last_time = Time.now.to_i
-    @mutex = Mutex.new
-    #init_db(@uri.split('//',2)[0])
-    connect {
-      # connection test
-    }
-    @sql = <<SQL
-SELECT id, timeout, data, created_at, resource, max_running/running AS weight
-FROM `#{@table}`
-LEFT JOIN (
-  SELECT resource AS res, COUNT(1) AS running
-  FROM `#{@table}` AS T
-  WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
-  GROUP BY resource
-) AS R ON resource = res
-WHERE timeout <= ? AND (max_running-running IS NULL OR max_running-running > 0)
-ORDER BY weight IS NOT NULL, weight DESC, timeout ASC
-LIMIT #{MAX_SELECT_ROW}
-SQL
-    # sqlite doesn't support SELECT ... FOR UPDATE but
-    # sqlite doesn't need it because the db is not shared
-    unless @uri.split('//',2)[0].to_s.include?('sqlite')
-      @sql << 'FOR UPDATE'
-    end
-  end
-
-  def create_tables
-    sql = ''
-    sql << "CREATE TABLE IF NOT EXISTS `#{@table}` ("
-    sql << "  id VARCHAR(256) NOT NULL,"
-    sql << "  timeout INT NOT NULL,"
-    sql << "  data BLOB NOT NULL,"
-    sql << "  created_at INT,"
-    sql << "  resource VARCHAR(256),"
-    sql << "  max_running INT,"
-    sql << "  PRIMARY KEY (id)"
-    sql << ");"
-    # TODO index
-    connect {
-      @db.run sql
-    }
-  end
-
-  private
-  def connect(&block)
-    #now = Time.now.to_i
-    @mutex.synchronize do
-      #if now - @last_time > KEEPALIVE
-      #  @db.disconnect
-      #end
-      #@last_time = now
-      retry_count = 0
-      begin
-        block.call
-      rescue
-        # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
-        if $!.to_s.include?('try restarting transaction')
-          err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
-          retry_count += 1
-          if retry_count < MAX_RETRY
-            STDERR.puts err + "\n  retrying."
-            sleep 0.5
-            retry
-          else
-            STDERR.puts err + "\n  abort."
-          end
-        end
-        raise
-      ensure
-        @db.disconnect
-      end
-    end
-  end
-
-  public
-  def list(&block)
-    @db.fetch("SELECT id, timeout, data, created_at, resource FROM `#{@table}` WHERE created_at IS NOT NULL ORDER BY timeout ASC;") {|row|
-      yield row[:id], row[:created_at], row[:data], row[:timeout], row[:resource]
-    }
-  end
-
-  MAX_SELECT_ROW = 8
-  #KEEPALIVE = 10
-  MAX_RETRY = 10
-
-  def acquire(timeout, now=Time.now.to_i)
-    connect {
-      while true
-        rows = 0
-        @db.transaction do
-          @db.fetch(@sql, now, now) {|row|
-            unless row[:created_at]
-              # finished/canceled task
-              @db["DELETE FROM `#{@table}` WHERE id=?;", row[:id]].delete
-
-            else
-              ## optimistic lock is not needed because the row is locked for update
-              #n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND timeout=?", timeout, row[:id], row[:timeout]].update
-              n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=?", timeout, row[:id]].update
-              if n > 0
-                return row[:id], Task.new(row[:id], row[:created_at], row[:data], row[:resource])
-              end
-            end
-
-            rows += 1
-          }
-        end
-        break nil if rows < MAX_SELECT_ROW
-      end
-    }
-  end
-
-  def finish(id, delete_timeout=3600, now=Time.now.to_i)
-    connect {
-      n = @db["UPDATE `#{@table}` SET timeout=?, created_at=NULL, resource=NULL WHERE id=? AND created_at IS NOT NULL;", now+delete_timeout-DELETE_OFFSET, id].update
-      return n > 0
-    }
-  end
-
-  def update(id, timeout)
-    connect {
-      n = @db["UPDATE `#{@table}` SET timeout=? WHERE id=? AND created_at IS NOT NULL;", timeout, id].update
-      if n <= 0
-        raise CanceledError, "Task id=#{id} is canceled."
-      end
-      return nil
-    }
-  end
-
-  def cancel(id, delete_timeout=3600, now=Time.now.to_i)
-    finish(id, delete_timeout, now)
-  end
-
-  def submit(id, data, time=Time.now.to_i, resource=nil, max_running=nil)
-    connect {
-      begin
-        data = Sequel::SQL::Blob.new(data)
-        n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at, resource, max_running) VALUES (?, ?, ?, ?, ?, ?);", id, time, data, time, resource, max_running].insert
-        return true
-      rescue Sequel::DatabaseError => e
-        # Sequel doesn't provide error classes to distinguish duplicate-entry from other
-        # errors like connectivity error. This code assumes the driver is mysql2 and
-        # the error message is "Mysql::ServerError::DupEntry: Duplicate entry"
-        if /: Duplicate entry/ =~ e.to_s
-          return nil
-        end
-        raise e
-      end
-    }
-  end
-end
-
-
-end
-

data/lib/perfectqueue/backend/simpledb.rb

@@ -1,139 +0,0 @@
-
-module PerfectQueue
-
-
-class SimpleDBBackend < Backend
-  def initialize(key_id, secret_key, domain)
-    require 'aws-sdk'
-    @consistent_read = false
-
-    @db = AWS::SimpleDB.new(
-      :access_key_id => key_id,
-      :secret_access_key => secret_key)
-
-    @domain_name = domain
-    @domain = @db.domains[@domain_name]
-    unless @domain.exists?
-      @domain = @db.domains.create(@domain_name)
-    end
-  end
-
-  attr_accessor :consistent_read
-
-  def use_consistent_read(b=true)
-    @consistent_read = b
-    self
-  end
-
-  def list(&block)
-    # TODO support resource limit
-    @domain.items.select('timeout', 'data', 'created_at',
-                        :where => "created_at != '' AND timeout > '#{int_encode(0)}'",
-                        :order => [:timeout, :asc],
-                        :consistent_read => @consistent_read) {|itemdata|
-      id = itemdata.name
-      attrs = itemdata.attributes
-      next unless attrs['created_at'].first
-      created_at = int_decode(attrs['created_at'].first)
-      data = attrs['data'].first
-      timeout = int_decode(attrs['timeout'].first)
-      yield id, created_at, data, timeout, nil
-    }
-  end
-
-  MAX_SELECT_ROW = 4
-
-  def acquire(timeout, now=Time.now.to_i)
-    while true
-      # TODO support resource limit
-      rows = 0
-      @domain.items.select('timeout', 'data', 'created_at',
-                          :where => "timeout <= '#{int_encode(now)}'",
-                          :order => [:timeout, :asc],
-                          :consistent_read => @consistent_read,
-                          :limit => MAX_SELECT_ROW) {|itemdata|
-        begin
-          id = itemdata.name
-          attrs = itemdata.attributes
-          salt = attrs['created_at'].first
-
-          if !salt || salt.empty?
-            # finished/canceled task
-            @domain.items[id].delete(:if=>{'created_at'=>''})
-
-          else
-            created_at = int_decode(salt)
-            @domain.items[id].attributes.replace('timeout'=>int_encode(timeout),
-                :if=>{'timeout'=>attrs['timeout'].first})
-
-            data = attrs['data'].first
-
-            return [id,salt], Task.new(id, created_at, data, nil)
-          end
-
-        rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
-        end
-
-        rows += 1
-      }
-      if rows < MAX_SELECT_ROW
-        return nil
-      end
-    end
-  end
-
-  def finish(token, delete_timeout=3600, now=Time.now.to_i)
-    begin
-      id, salt = *token
-      @domain.items[id].attributes.replace('timeout'=>int_encode(now+delete_timeout), 'created_at'=>'',
-          :if=>{'created_at'=>salt})
-      return true
-    rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
-      return false
-    end
-  end
-
-  def update(token, timeout)
-    begin
-      id, salt = *token
-      @domain.items[id].attributes.replace('timeout'=>int_encode(timeout),
-          :if=>{'created_at'=>salt})
-    rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::AttributeDoesNotExist
-      raise CanceledError, "Task id=#{id} is canceled."
-    end
-    nil
-  end
-
-  def cancel(id, delete_timeout=3600, now=Time.now.to_i)
-    salt = @domain.items[id].attributes['created_at'].first
-    unless salt
-      return false
-    end
-    token = [id,salt]
-    finish(token, delete_timeout, now)
-  end
-
-  def submit(id, data, time=Time.now.to_i, resource=nil)
-    # TODO support resource limit
-    begin
-      @domain.items[id].attributes.replace('timeout'=>int_encode(time), 'created_at'=>int_encode(time), 'data'=>data,
-          :unless=>'timeout')
-      return true
-    rescue AWS::SimpleDB::Errors::ConditionalCheckFailed, AWS::SimpleDB::Errors::ExistsAndExpectedValue
-      return nil
-    end
-  end
-
-  private
-  def int_encode(num)
-    "%08x" % num
-  end
-
-  def int_decode(str)
-    str.to_i(16)
-  end
-end
-
-
-end
-

data/test/backend_test.rb

@@ -1,259 +0,0 @@
-require File.dirname(__FILE__)+'/test_helper'
-
-class BackendTest < Test::Unit::TestCase
-  TIMEOUT = 10
-  DB_PATH = File.dirname(__FILE__)+'/test.db'
-  DB_URI = "sqlite://#{DB_PATH}"
-
-  def clean_backend
-    @key_prefix = "test-#{"%08x"%rand(2**32)}-"
-    db = open_backend
-    db.list {|id,created_at,data,timeout|
-      db.cancel(id)
-    }
-    FileUtils.rm_f DB_PATH
-  end
-
-  def open_backend
-    #PerfectQueue::SimpleDBBackend.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], 'perfectqueue-test-1').use_consistent_read
-    db = PerfectQueue::RDBBackend.new(DB_URI, "perfectdb_test")
-    db.create_tables
-    db
-  end
-
-  it 'acquire' do
-    clean_backend
-
-    db1 = open_backend
-    db2 = open_backend
-    db3 = open_backend
-
-    time = Time.now.to_i
-
-    ok = db1.submit(@key_prefix+'test1', 'data1', time)
-    assert_equal true, ok
-
-    token, task = db2.acquire(time+TIMEOUT)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test1', task.id
-    assert_equal time, task.created_at
-    assert_equal 'data1', task.data
-
-    token_, task_ = db3.acquire(time+TIMEOUT)
-    assert_equal nil, token_
-  end
-
-  it 'finish' do
-    clean_backend
-
-    db1 = open_backend
-    db2 = open_backend
-
-    time = Time.now.to_i
-
-    ok = db1.submit(@key_prefix+'test1', 'data1', time)
-    assert_equal true, ok
-
-    token, task = db1.acquire(time+TIMEOUT)
-    assert_not_equal nil, task
-
-    ok = db1.finish(token)
-    assert_equal true, ok
-
-    token_, task_ = db2.acquire(time+TIMEOUT)
-    assert_equal nil, token_
-
-    ok = db1.finish(token)
-    assert_equal false, ok
-  end
-
-  it 'canceled' do
-    clean_backend
-
-    db1 = open_backend
-    db2 = open_backend
-
-    time = Time.now.to_i
-
-    ok = db1.submit(@key_prefix+'test1', 'data1', time)
-    assert_equal true, ok
-
-    token, task = db1.acquire(time+TIMEOUT)
-    assert_not_equal nil, task
-
-    ok = db1.cancel(task.id)
-    assert_equal true, ok
-
-    token_, task_ = db2.acquire(time+TIMEOUT)
-    assert_equal nil, token_
-
-    assert_raise(PerfectQueue::CanceledError) do
-      db1.update(token, time+TIMEOUT)
-    end
-
-    ok = db1.cancel(task.id)
-    assert_equal false, ok
-  end
-
-  it 'order' do
-    clean_backend
-
-    db1 = open_backend
-    db2 = open_backend
-    db3 = open_backend
-
-    time = Time.now.to_i
-
-    ok = db1.submit(@key_prefix+'test1', 'data1', time)
-    assert_equal true, ok
-
-    ok = db1.submit(@key_prefix+'test2', 'data2', time-1)
-    assert_equal true, ok
-
-    ok = db1.submit(@key_prefix+'test3', 'data3', time+1)
-    assert_equal true, ok
-
-    token, task = db2.acquire(time+TIMEOUT, time+1)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test2', task.id
-    assert_equal time-1, task.created_at
-    assert_equal 'data2', task.data
-
-    token, task = db2.acquire(time+TIMEOUT, time+1)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test1', task.id
-    assert_equal time, task.created_at
-    assert_equal 'data1', task.data
-
-    token, task = db2.acquire(time+TIMEOUT, time+1)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test3', task.id
-    assert_equal time+1, task.created_at
-    assert_equal 'data3', task.data
-  end
-
-  it 'timeout' do
-    clean_backend
-
-    db1 = open_backend
-    db2 = open_backend
-
-    time = Time.now.to_i
-
-    ok = db1.submit(@key_prefix+'test1', 'data1', time)
-    assert_equal true, ok
-
-    token, task = db1.acquire(time+TIMEOUT)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test1', task.id
-    assert_equal time, task.created_at
-    assert_equal 'data1', task.data
-
-    token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test1', task.id
-    assert_equal time, task.created_at
-    assert_equal 'data1', task.data
-  end
-
-  it 'extend' do
-    clean_backend
-
-    db1 = open_backend
-    db2 = open_backend
-
-    time = Time.now.to_i
-
-    ok = db1.submit(@key_prefix+'test1', 'data1', time)
-    assert_equal true, ok
-
-    token, task = db1.acquire(time+TIMEOUT)
-    assert_not_equal nil, task
-
-    assert_nothing_raised do
-      db1.update(token, time+TIMEOUT+1)
-    end
-
-    token_, task_ = db2.acquire(time+TIMEOUT, time)
-    assert_equal nil, token_
-
-    token, task = db2.acquire(time+TIMEOUT*2, time+TIMEOUT+1)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test1', task.id
-    assert_equal time, task.created_at
-    assert_equal 'data1', task.data
-  end
-
-  it 'release' do
-    clean_backend
-
-    db1 = open_backend
-    db2 = open_backend
-
-    time = Time.now.to_i
-
-    ok = db1.submit(@key_prefix+'test1', 'data1', time)
-    assert_equal true, ok
-
-    token, task = db1.acquire(time+TIMEOUT)
-    assert_not_equal nil, task
-
-    assert_nothing_raised do
-      db1.update(token, time+TIMEOUT+1)
-    end
-
-    token_, task_ = db2.acquire(time+TIMEOUT, time)
-    assert_equal nil, token_
-
-    assert_nothing_raised do
-      db1.update(token, time)
-    end
-
-    token, task = db2.acquire(time+TIMEOUT, time)
-    assert_not_equal nil, task
-    assert_equal @key_prefix+'test1', task.id
-    assert_equal time, task.created_at
-    assert_equal 'data1', task.data
-  end
-
-  it 'resource limit' do
-    clean_backend
-
-    db1 = open_backend
-
-    time = Time.now.to_i
-
-    3.times do |i|
-      ok = db1.submit(@key_prefix+'test'+i.to_s, 'data1', time-(i+1), 'user1', 2)
-      assert_equal true, ok
-    end
-    ok = db1.submit(@key_prefix+'test5', 'data2', time, 'user2', 2)
-    assert_equal true, ok
-
-    token_1 = nil
-    task_1 = nil
-
-    token_1, task_1 = db1.acquire(time+TIMEOUT, time)
-    assert_not_equal nil, task_1
-    assert_equal "user1", task_1.resource
-
-    token, task = db1.acquire(time+TIMEOUT, time)
-    assert_not_equal nil, task
-    assert_equal "user2", task.resource
-
-    token, task = db1.acquire(time+TIMEOUT, time)
-    assert_not_equal nil, task
-    assert_equal "user1", task.resource
-
-    token, task = db1.acquire(time+TIMEOUT, time)
-    assert_equal nil, task
-
-    ok = db1.finish(token_1)
-    assert_equal true, ok
-
-    token, task = db1.acquire(time+TIMEOUT, time)
-    assert_not_equal nil, task
-    assert_equal "user1", task.resource
-  end
-end
-