perfectqueue 0.8.5 → 0.8.6

data/ChangeLog

@@ -1,4 +1,11 @@
 
+== 2012-07-23 version 0.8.6
+
+* rdb_backend supports 'max_running' which is added in v0.7.21
+* Fixed DaemonsLogger to use File::CREAT flag to open a file
+* Show version when worker starts
+
+
 == 2012-07-06 version 0.8.5
 
 * Introduced PerfectQueue::Application::Decider
@@ -34,6 +41,16 @@ in case it was called within 1 second
 * Redesigned API
 
 
+== 2012-07-19 version 0.7.22
+
+* Fixed RDBBackend SQL
+
+
+== 2012-07-19 version 0.7.21
+
+* Added max_running column for fair resource restriction
+
+
 == 2012-01-24 version 0.7.20
 
 * Fixed MonitorThread

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -43,8 +43,8 @@ module PerfectQueue
           # connection test
         }
 
-        @sql = <<SQL
-SELECT id, timeout, data, created_at, resource
+    @sql = <<SQL
+SELECT id, timeout, data, created_at, resource, max_running-running AS runnable
 FROM `#{@table}`
 LEFT JOIN (
   SELECT resource AS res, COUNT(1) AS running
@@ -52,8 +52,9 @@ LEFT JOIN (
   WHERE timeout > ? AND created_at IS NOT NULL AND resource IS NOT NULL
   GROUP BY resource
 ) AS R ON resource = res
-WHERE timeout <= ? AND (running IS NULL OR running < #{MAX_RESOURCE})
-ORDER BY timeout ASC LIMIT #{MAX_SELECT_ROW}
+WHERE timeout <= ? AND (max_running-running IS NULL OR max_running-running > 0)
+ORDER BY runnable IS NOT NULL, runnable DESC, timeout ASC
+LIMIT #{MAX_SELECT_ROW}
 SQL
 
         # sqlite doesn't support SELECT ... FOR UPDATE but
@@ -78,6 +79,7 @@ SQL
               data BLOB NOT NULL,
               created_at INT,
               resource VARCHAR(256),
+              max_running INT,
               PRIMARY KEY (id)
             );]
         connect {
@@ -122,14 +124,17 @@ SQL
       def submit(key, type, data, options)
         now = (options[:now] || Time.now).to_i
         run_at = (options[:run_at] || now).to_i
-        user = options[:user]
-        priority = options[:priority]  # not supported
+        user = options[:user].to_s
+        max_running = options[:max_running]
         data = data ? data.dup : {}
         data['type'] = type
 
         connect {
           begin
-            n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at, resource) VALUES (?, ?, ?, ?, ?);", key, run_at, data.to_json, now, user].insert
+            n = @db[
+              "INSERT INTO `#{@table}` (id, timeout, data, created_at, resource, max_running) VALUES (?, ?, ?, ?, ?, ?);",
+              key, run_at, data.to_json, now, user, max_running
+            ].insert
             return Task.new(@client, key)
           rescue Sequel::DatabaseError
             raise IdempotentAlreadyExistsError, "task key=#{key} already exists"

data/lib/perfectqueue/daemons_logger.rb

@@ -26,7 +26,7 @@ module PerfectQueue
       @stderr_hook = false
       if dev.is_a?(String)
         @path = dev
-        @io = File.open(dev, File::WRONLY|File::APPEND)
+        @io = File.open(@path, File::WRONLY|File::APPEND|File::CREAT)
       else
         @io = dev
       end

data/lib/perfectqueue/signal_queue.rb

@@ -95,7 +95,7 @@ module PerfectQueue
 
     private
     def enqueue(sig)
-      if Thread.current == self
+      if Thread.current == @thread
         @queue << sig
         if @mutex.locked?
           @cond.signal

data/lib/perfectqueue/task_metadata.rb

@@ -78,6 +78,8 @@ module PerfectQueue
   end
 
   class TaskMetadata
+    include Model
+
     def initialize(client, key, attributes)
       super(client)
       @key = key

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.5"
+  VERSION = "0.8.6"
 end

data/lib/perfectqueue/worker.rb

@@ -31,21 +31,20 @@ module PerfectQueue
       @runner = runner
       block = Proc.new { config } if config
       @config_load_proc = block
-      @finished = false
     end
 
     def run
-      @sig = install_signal_handlers
-      begin
+      @log.info "PerfectQueue #{VERSION}"
+
+      install_signal_handlers do
         @engine = Engine.new(@runner, load_config)
         begin
           @engine.run
         ensure
           @engine.shutdown(true)
         end
-      ensure
-        @sig.shutdown
       end
+
       return nil
     rescue
       @log.error "#{$!.class}: #{$!}"
@@ -112,8 +111,8 @@ module PerfectQueue
       return config
     end
 
-    def install_signal_handlers
-      SignalQueue.start do |sig|
+    def install_signal_handlers(&block)
+      sig = SignalQueue.start do |sig|
         sig.trap :TERM do
           stop(false)
         end
@@ -147,6 +146,12 @@ module PerfectQueue
 
         trap :CHLD, "SIG_IGN"
       end
+
+      begin
+        block.call
+      ensure
+        sig.shutdown
+      end
     end
   end
 

data/spec/queue_spec.rb

@@ -1,13 +1,7 @@
 require 'spec_helper'
 
 describe Queue do
-  let :queue do
-    create_test_queue
-  end
-
-  after do
-    queue.client.close
-  end
+  include QueueTest
 
   it 'is a Queue' do
     queue.class.should == PerfectQueue::Queue

data/spec/rdb_compat_backend_spec.rb

@@ -36,5 +36,35 @@ describe Backend::RDBCompatBackend do
     t.type.should == 'query'
     t.key.should == 'query.379474'
   end
+
+  it 'resource limit' do
+    time = Time.now.to_i
+
+    3.times do |i|
+      queue.submit("test_#{i}", 'user01', {}, :now=>time-(i+1), :user=>'u1', :max_running=>2)
+    end
+    queue.submit("test_5", 'user02', {}, :now=>time, :user=>'u2', :max_running=>2)
+
+    task1 = queue.poll(:now=>time+10)
+    task1.should_not == nil
+    task1.type.should == 'user01'
+
+    task2 = queue.poll(:now=>time+10)
+    task2.should_not == nil
+    task2.type.should == 'user02'
+
+    task3 = queue.poll(:now=>time+10)
+    task3.should_not == nil
+    task3.type.should == 'user01'
+
+    task4 = queue.poll(:now=>time+10)
+    task4.should == nil
+
+    task1.finish!
+
+    task5 = queue.poll(:now=>time+10)
+    task5.should_not == nil
+    task5.type.should == 'user01'
+  end
 end
 

data/spec/spec_helper.rb

@@ -13,32 +13,38 @@ end
 
 require 'fileutils'
 
-def test_queue_config
-  {:type=>'rdb_compat', :url=>'sqlite://spec/test.db', :table=>'test_tasks', :processor_type=>'thread'}
-end
-
-def create_test_queue
-  FileUtils.rm_f 'spec/test.db'
-  queue = PerfectQueue.open(test_queue_config)
-
-  sql = %[
-      CREATE TABLE IF NOT EXISTS `test_tasks` (
-        id VARCHAR(256) NOT NULL,
-        timeout INT NOT NULL,
-        data BLOB NOT NULL,
-        created_at INT,
-        resource VARCHAR(256),
-        PRIMARY KEY (id)
-      );]
-
-  queue.client.backend.db.run sql
-
-  return queue
+module QueueTest
+  def self.included(mod)
+    mod.module_eval do
+      let :database_path do
+        'spec/test.db'
+      end
+
+      let :queue_config do
+        {
+          :type => 'rdb_compat',
+          :url => "sqlite://#{database_path}",
+          :table => 'test_tasks',
+          :processor_type => 'thread'
+        }
+      end
+
+      let :queue do
+        PerfectQueue.open(queue_config)
+      end
+
+      before do
+        FileUtils.rm_f database_path
+        queue.client.init_database
+      end
+
+      after do
+        queue.close
+      end
+    end
+  end
 end
 
-def get_test_queue
-  PerfectQueue.open(test_queue_config)
-end
 
 include PerfectQueue
 

data/spec/stress.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe Queue do
+  include QueueTest
+
+  let :thread_num do
+    5
+  end
+
+  let :loop_num do
+    50
+  end
+
+  let :now do
+    Time.now.to_i
+  end
+
+  def thread_main
+    thread_id = Thread.current.object_id
+
+    loop_num.times do |i|
+      queue.submit("#{thread_id}-#{i}", "type01", {}, :now=>now-10)
+      task = queue.poll(:now=>now, :alive_time=>60)
+      task.should_not == nil
+      task.heartbeat!(:now=>now, :alive_time=>70)
+      task.finish!(:now=>now, :retention_time=>80)
+    end
+  end
+
+  it 'stress' do
+    puts "stress test with threads=#{thread_num} * loop_num=#{loop_num} = #{thread_num * loop_num} tasks"
+
+    # initialize queue here
+    queue
+    now
+
+    start_at = Time.now
+    (1..thread_num).map {
+      Thread.new(&method(:thread_main))
+    }.each {|thread|
+      thread.join
+    }
+    finish_at = Time.now
+
+    elapsed = finish_at - start_at
+    task_num = thread_num * loop_num
+    puts "#{elapsed} sec."
+    puts "#{task_num / elapsed} req/sec."
+    puts "#{elapsed / task_num} sec/req."
+  end
+
+end
+

data/spec/worker_spec.rb

@@ -29,9 +29,10 @@ class TestApp < PerfectQueue::Application::Dispatch
 end
 
 describe Worker do
+  include QueueTest
+
   before do
-    create_test_queue.close
-    @worker = Worker.new(TestApp, test_queue_config)
+    @worker = Worker.new(TestApp, queue_config)
     @thread = Thread.new {
       @worker.run
     }
@@ -42,17 +43,11 @@ describe Worker do
     @thread.join
   end
 
-  def submit(*args)
-    queue = get_test_queue
-    queue.submit(*args)
-    queue.close
-  end
-
   it 'route' do
     TestHandler.any_instance.should_receive(:run).once
     RegexpHandler.any_instance.should_receive(:run).once
-    submit('task01', 'test', {})
-    submit('task02', 'reg01', {})
+    queue.submit('task01', 'test', {})
+    queue.submit('task02', 'reg01', {})
     sleep 1
   end
 
@@ -72,7 +67,7 @@ describe Worker do
 
   it 'kill reason' do
     TestHandler.any_instance.should_receive(:kill).once #.with(kind_of(PerfectQueue::CancelRequestedError))  # FIXME 'with' dead locks
-    submit('task01', 'test', {'sleep'=>4})
+    queue.submit('task01', 'test', {'sleep'=>4})
     sleep 2
     Process.kill(:TERM, Process.pid)
     @thread.join

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-07 00:00:00.000000000Z
+date: 2012-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: &70242051861300 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 3.26.0
   type: :runtime
   prerelease: false
-  version_requirements: *70242051861300
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.26.0
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70242051860640 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +37,15 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70242051860640
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70242051859840 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +53,15 @@ dependencies:
         version: 2.10.0
   type: :development
   prerelease: false
-  version_requirements: *70242051859840
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.10.0
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70242051858860 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +69,15 @@ dependencies:
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *70242051858860
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.4
 - !ruby/object:Gem::Dependency
   name: sqlite3
-  requirement: &70242051858120 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +85,12 @@ dependencies:
         version: 1.3.3
   type: :development
   prerelease: false
-  version_requirements: *70242051858120
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.3
 description: Highly available distributed cron built on RDBMS
 email: frsyuki@gmail.com
 executables:
@@ -112,6 +137,7 @@ files:
 - spec/queue_spec.rb
 - spec/rdb_compat_backend_spec.rb
 - spec/spec_helper.rb
+- spec/stress.rb
 - spec/worker_spec.rb
 homepage: https://github.com/treasure-data/perfectqueue
 licenses: []
@@ -133,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.12
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Highly available distributed cron built on RDBMS
@@ -141,5 +167,6 @@ test_files:
 - spec/queue_spec.rb
 - spec/rdb_compat_backend_spec.rb
 - spec/spec_helper.rb
+- spec/stress.rb
 - spec/worker_spec.rb
 has_rdoc: false