perfectqueue 0.8.0 → 0.8.1

data/ChangeLog

@@ -1,4 +1,18 @@
 
+== 2012-06-25 version 0.8.1
+
+* Added autoload for PerfectQueue::VERSION
+* Fixed backward compatiblity of the 'type' field in rdb_compat backend
+* Added IdempotentAlreadyFinishedError and IdempotentAlreadyExistsError
+  which should be ignored for idempotency
+
+
+== 2012-05-23 version 0.8.0
+
+* New major version
+* Redesigned API
+
+
 == 2012-01-24 version 0.7.20
 
 * Fixed MonitorThread

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -90,7 +90,7 @@ SQL
         now = (options[:now] || Time.now).to_i
 
         connect {
-          row = @db.fetch("SELECT timeout, data, created_at, resource FROM `#{@table}` LIMIT 1").first
+          row = @db.fetch("SELECT timeout, data, created_at, resource FROM `#{@table}` WHERE id=? LIMIT 1", key).first
           unless row
             raise NotFoundError, "task key=#{key} does no exist"
           end
@@ -124,7 +124,7 @@ SQL
         run_at = (options[:run_at] || now).to_i
         user = options[:user]
         priority = options[:priority]  # not supported
-        data ||= {}
+        data = data ? data.dup : {}
         data['type'] = type
 
         connect {
@@ -132,7 +132,7 @@ SQL
             n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at, resource) VALUES (?, ?, ?, ?, ?);", key, run_at, data.to_json, now, user].insert
             return Task.new(@client, key)
           rescue Sequel::DatabaseError
-            raise AlreadyExistsError, "task key=#{key} already exists"
+            raise IdempotentAlreadyExistsError, "task key=#{key} already exists"
           end
         }
       end
@@ -198,7 +198,7 @@ SQL
         connect {
           n = @db["UPDATE `#{@table}` SET timeout=?, created_at=NULL, resource=NULL WHERE id=? AND created_at IS NOT NULL;", delete_timeout, key].update
           if n <= 0
-            raise AlreadyFinishedError, "task key=#{key} does not exist or already finished."
+            raise IdempotentAlreadyFinishedError, "task key=#{key} does not exist or already finished."
           end
         }
         nil
@@ -272,13 +272,21 @@ SQL
           status = TaskStatus::RUNNING
         end
 
-        begin
-          data = JSON.parse(row[:data] || '{}')
-        rescue
+        d = row[:data]
+        if d == nil || d == ''
           data = {}
+        else
+          begin
+            data = JSON.parse(d)
+          rescue
+            data = {}
+          end
         end
 
-        type = data['type'] || ''
+        type = data.delete('type')
+        if type == nil || type.empty?
+          type = row[:id].split(/\./, 2)[0]
+        end
 
         attributes = {
           :status => status,

data/lib/perfectqueue/error.rb

@@ -49,5 +49,21 @@ module PerfectQueue
 
   class GracefulProcessStopError < ProcessStopError
   end
+
+  # Applications can ignore these errors to achieve idempotency
+  module IdempotentError
+  end
+
+  class IdempotentAlreadyFinishedError < AlreadyFinishedError
+    include IdempotentError
+  end
+
+  class IdempotentAlreadyExistsError < AlreadyExistsError
+    include IdempotentError
+  end
+
+  class IdempotentNotFoundError < NotFoundError
+    include IdempotentError
+  end
 end
 

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.0"
+  VERSION = "0.8.1"
 end

data/lib/perfectqueue.rb

@@ -40,6 +40,7 @@ module PerfectQueue
     :TaskStatus => 'perfectqueue/task_status',
     :Worker => 'perfectqueue/worker',
     :SignalQueue => 'perfectqueue/signal_queue',
+    :VERSION => 'perfectqueue/version',
   }.each_pair {|k,v|
     autoload k, File.expand_path(v, File.dirname(__FILE__))
   }

data/spec/queue_spec.rb

@@ -1,44 +1,44 @@
 require 'spec_helper'
 
 describe Queue do
-  before do
-    @queue = create_test_queue
+  let :queue do
+    create_test_queue
   end
 
   after do
-    @queue.client.close
+    queue.client.close
   end
 
   it 'is a Queue' do
-    @queue.class.should == PerfectQueue::Queue
+    queue.class.should == PerfectQueue::Queue
   end
 
   it 'succeess submit' do
-    @queue.submit('task01', 'type1', {})
+    queue.submit('task01', 'type1', {})
   end
 
   it 'fail duplicated submit' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {}, :now=>now)
+    queue.submit('task01', 'type1', {}, :now=>now)
 
     lambda {
-      @queue.submit('task01', 'type1', {}, :now=>now+1)
+      queue.submit('task01', 'type1', {}, :now=>now+1)
     }.should raise_error AlreadyExistsError
 
-    @queue['task01'].cancel_request!(:now=>now+2)
+    queue['task01'].cancel_request!(:now=>now+2)
 
     lambda {
-      @queue.submit('task01', 'type1', {}, :now=>now+10)
+      queue.submit('task01', 'type1', {}, :now=>now+10)
     }.should raise_error AlreadyExistsError
   end
 
   it 'list' do
-    @queue.submit('task01', 'type1', {"a"=>1})
-    @queue.submit('task02', 'type1', {"a"=>2})
-    @queue.submit('task03', 'type1', {"a"=>3})
+    queue.submit('task01', 'type1', {"a"=>1})
+    queue.submit('task02', 'type1', {"a"=>2})
+    queue.submit('task03', 'type1', {"a"=>3})
 
     a = []
-    @queue.each {|t| a << t }
+    queue.each {|t| a << t }
     a.sort_by! {|t| t.key }
 
     task01 = a.shift
@@ -64,170 +64,170 @@ describe Queue do
 
   it 'poll' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
-    @queue.submit('task02', 'type1', {"a"=>2}, :now=>now+1)
-    @queue.submit('task03', 'type1', {"a"=>3}, :now=>now+2)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task02', 'type1', {"a"=>2}, :now=>now+1)
+    queue.submit('task03', 'type1', {"a"=>3}, :now=>now+2)
 
-    task01 = @queue.poll(:now=>now+10)
+    task01 = queue.poll(:now=>now+10)
     task01.key.should == 'task01'
 
-    t2 = @queue.poll(:now=>now+10)
+    t2 = queue.poll(:now=>now+10)
     t2.key.should == 'task02'
 
-    t3 = @queue.poll(:now=>now+10)
+    t3 = queue.poll(:now=>now+10)
     t3.key.should == 'task03'
 
-    t4 = @queue.poll(:now=>now+10)
+    t4 = queue.poll(:now=>now+10)
     t4.should == nil
   end
 
   it 'release' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
-    task01 = @queue.poll(:now=>now+10)
+    task01 = queue.poll(:now=>now+10)
     task01.key.should == 'task01'
 
-    t2 = @queue.poll(:now=>now+10)
+    t2 = queue.poll(:now=>now+10)
     t2.should == nil
 
     task01.release!(:now=>now+10)
 
-    t3 = @queue.poll(:now=>now+11)
+    t3 = queue.poll(:now=>now+11)
     t3.key.should == 'task01'
   end
 
   it 'timeout' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
-    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01 = queue.poll(:now=>now+10, :alive_time=>10)
     task01.key.should == 'task01'
 
-    t2 = @queue.poll(:now=>now+15)
+    t2 = queue.poll(:now=>now+15)
     t2.should == nil
 
-    t3 = @queue.poll(:now=>now+20)
+    t3 = queue.poll(:now=>now+20)
     t3.key.should == 'task01'
   end
 
   it 'heartbeat' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
-    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01 = queue.poll(:now=>now+10, :alive_time=>10)
     task01.key.should == 'task01'
 
     task01.heartbeat!(:alive_time=>15, :now=>now+10)
 
-    t2 = @queue.poll(:now=>now+20)
+    t2 = queue.poll(:now=>now+20)
     t2.should == nil
 
-    t3 = @queue.poll(:now=>now+30)
+    t3 = queue.poll(:now=>now+30)
     t3.key.should == 'task01'
   end
 
   it 'retry' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
-    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01 = queue.poll(:now=>now+10, :alive_time=>10)
     task01.key.should == 'task01'
 
     task01.retry!(:retry_wait=>15, :now=>now+10)
 
-    t2 = @queue.poll(:now=>now+20)
+    t2 = queue.poll(:now=>now+20)
     t2.should == nil
 
-    t3 = @queue.poll(:now=>now+30)
+    t3 = queue.poll(:now=>now+30)
     t3.key.should == 'task01'
   end
 
   it 'froce_finish' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
-    task01 = @queue.poll(:now=>now+10)
+    task01 = queue.poll(:now=>now+10)
     task01.key.should == 'task01'
 
-    @queue['task01'].metadata.running?.should == true
+    queue['task01'].metadata.running?.should == true
 
-    @queue['task01'].force_finish!(:now=>now+11)
+    queue['task01'].force_finish!(:now=>now+11)
 
-    @queue['task01'].metadata.finished?.should == true
+    queue['task01'].metadata.finished?.should == true
   end
 
   it 'status' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
     # rdb_backend backend can't distinguish running with waiting
-    #@queue['task01'].metadata.finished?.should == false
-    #@queue['task01'].metadata.running?.should == false
-    #@queue['task01'].metadata.waiting?.should == true
-    #@queue['task01'].metadata.cancel_requested?.should == false
+    #queue['task01'].metadata.finished?.should == false
+    #queue['task01'].metadata.running?.should == false
+    #queue['task01'].metadata.waiting?.should == true
+    #queue['task01'].metadata.cancel_requested?.should == false
 
-    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01 = queue.poll(:now=>now+10, :alive_time=>10)
     task01.key.should == 'task01'
 
-    @queue['task01'].metadata.finished?.should == false
-    @queue['task01'].metadata.running?.should == true
-    @queue['task01'].metadata.waiting?.should == false
-    @queue['task01'].metadata.cancel_requested?.should == false
+    queue['task01'].metadata.finished?.should == false
+    queue['task01'].metadata.running?.should == true
+    queue['task01'].metadata.waiting?.should == false
+    queue['task01'].metadata.cancel_requested?.should == false
 
     task01.cancel_request!
 
     # status of cancel_requested running tasks is cancel_requested
-    @queue['task01'].metadata.finished?.should == false
-    @queue['task01'].metadata.running?.should == false
-    @queue['task01'].metadata.waiting?.should == false
-    @queue['task01'].metadata.cancel_requested?.should == true
+    queue['task01'].metadata.finished?.should == false
+    queue['task01'].metadata.running?.should == false
+    queue['task01'].metadata.waiting?.should == false
+    queue['task01'].metadata.cancel_requested?.should == true
 
     task01.finish!
 
-    @queue['task01'].metadata.finished?.should == true
-    @queue['task01'].metadata.running?.should == false
-    @queue['task01'].metadata.waiting?.should == false
-    @queue['task01'].metadata.cancel_requested?.should == false
+    queue['task01'].metadata.finished?.should == true
+    queue['task01'].metadata.running?.should == false
+    queue['task01'].metadata.waiting?.should == false
+    queue['task01'].metadata.cancel_requested?.should == false
   end
 
   it 'fail canceling finished task' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
-    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01 = queue.poll(:now=>now+10, :alive_time=>10)
     task01.key.should == 'task01'
 
     task01.finish!
 
     lambda {
-      @queue['task01'].cancel_request!
+      queue['task01'].cancel_request!
     }.should raise_error AlreadyFinishedError
   end
 
   it 'retention_time' do
     now = Time.now.to_i
-    @queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
+    queue.submit('task01', 'type1', {"a"=>1}, :now=>now+0)
 
-    @queue['task01'].metadata.finished?.should == false
+    queue['task01'].metadata.finished?.should == false
 
-    task01 = @queue.poll(:now=>now+10, :alive_time=>10)
+    task01 = queue.poll(:now=>now+10, :alive_time=>10)
     task01.key.should == 'task01'
 
     task01.finish!(:now=>now+11, :retention_time=>10)
 
-    @queue.poll(:now=>now+12)
+    queue.poll(:now=>now+12)
 
-    @queue['task01'].exists?.should == true
+    queue['task01'].exists?.should == true
 
-    @queue.poll(:now=>now+22)
+    queue.poll(:now=>now+22)
 
-    @queue['task01'].exists?.should == false
+    queue['task01'].exists?.should == false
   end
 
   it 'get_task_metadata failed with NotFoundError' do
     lambda {
-      @queue['task99'].metadata
+      queue['task99'].metadata
     }.should raise_error NotFoundError
   end
 end

data/spec/rdb_compat_backend_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+require 'perfectqueue/backend/rdb_compat'
+
+describe Backend::RDBCompatBackend do
+  let :queue do
+    FileUtils.rm_f 'spec/test.db'
+    queue = PerfectQueue.open({:type=>'rdb_compat', :url=>'sqlite://spec/test.db', :table=>'test_tasks', :processor_type=>'thread'})
+    queue.client.init_database
+    queue
+  end
+
+  let :client do
+    queue.client
+  end
+
+  let :backend do
+    client.backend
+  end
+
+  it 'backward compatibility 1' do
+    backend.db["INSERT INTO test_tasks (id, timeout, data, created_at, resource) VALUES (?, ?, ?, ?, ?)", "merge_type.1339801200", 1339801201, {'url'=>nil}.to_json, 1339801201, "1"].insert
+    ts = backend.acquire(60, 1, {:now=>1339801203})
+    ts.should_not == nil
+    t = ts[0]
+    t.data.should == {'url'=>nil}
+    t.type.should == 'merge_type'
+    t.key.should == 'merge_type.1339801200'
+  end
+
+  it 'backward compatibility 2' do
+    backend.db["INSERT INTO test_tasks (id, timeout, data, created_at, resource) VALUES (?, ?, ?, ?, ?)", "query.379474", 1339801201, {'query_id'=>32}.to_json, 1339801201, nil].insert
+    ts = backend.acquire(60, 1, {:now=>1339801203})
+    ts.should_not == nil
+    t = ts[0]
+    t.data.should == {'query_id'=>32}
+    t.type.should == 'query'
+    t.key.should == 'query.379474'
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-23 00:00:00.000000000Z
+date: 2012-06-25 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70295369852020 !ruby/object:Gem::Requirement
+  requirement: &70128708982980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 3.26.0
   type: :runtime
   prerelease: false
-  version_requirements: *70295369852020
+  version_requirements: *70128708982980
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70295369850460 !ruby/object:Gem::Requirement
+  requirement: &70128708981280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70295369850460
+  version_requirements: *70128708981280
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70295369849140 !ruby/object:Gem::Requirement
+  requirement: &70128708976640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 2.10.0
   type: :development
   prerelease: false
-  version_requirements: *70295369849140
+  version_requirements: *70128708976640
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70295369848300 !ruby/object:Gem::Requirement
+  requirement: &70128708973780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *70295369848300
+  version_requirements: *70128708973780
 - !ruby/object:Gem::Dependency
   name: sqlite3
-  requirement: &70295369847400 !ruby/object:Gem::Requirement
+  requirement: &70128708967780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: 1.3.3
   type: :development
   prerelease: false
-  version_requirements: *70295369847400
+  version_requirements: *70128708967780
 description: Highly available distributed cron built on RDBMS
 email: frsyuki@gmail.com
 executables:
@@ -109,6 +109,7 @@ files:
 - lib/perfectqueue/worker.rb
 - perfectqueue.gemspec
 - spec/queue_spec.rb
+- spec/rdb_compat_backend_spec.rb
 - spec/spec_helper.rb
 - spec/worker_spec.rb
 homepage: https://github.com/treasure-data/perfectqueue
@@ -137,6 +138,7 @@ specification_version: 3
 summary: Highly available distributed cron built on RDBMS
 test_files:
 - spec/queue_spec.rb
+- spec/rdb_compat_backend_spec.rb
 - spec/spec_helper.rb
 - spec/worker_spec.rb
 has_rdoc: false