perfectqueue 0.8.47 → 0.8.48

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68a42f45b43150785b9c0a0ad1e6757da6b8104f
-  data.tar.gz: 64b1df2f881992eaf48f1811b323966c35cdfccd
+  metadata.gz: 56f251e9261948702f9f058179343788c08ea08e
+  data.tar.gz: 4bd8336b27bfc2097a22581120e872fe621d4c53
 SHA512:
-  metadata.gz: aa3effbb36e362da777ef15ab8aba0feb76471d150e7ae887cf299d90cf48d785f90b3f976907ad0e0fe6256ea7dd4bfa5198676e931a581778aae5c3dcfb677
-  data.tar.gz: 4235badb3ba5fdc97d570e14fed8c3d405c7b9d5fe98ee7962b7df3bc2bfef730a275aef241418d61fdc1d9a02e0b8401a8e5979a53af21c59428bd2be5ab5d3
+  metadata.gz: 0a826da88811b6f511b4e4b0a6bdab9255a0a792848874b761fb9c2b6014fb3ac32f1c2a8231c8bdd54dfec82bb37fbd46b87664d392f829fb9fadfcb4388d3e
+  data.tar.gz: 4591e5a7f253502dafb1687ce57d572fa8b7742b2482be6938d41ce501d9e42f44cc4a2ca3e9a13a4a7de6f50f3518831b2bc9ba60bbac5ec29930ea67c36602

data/.travis.yml

@@ -1,8 +1,7 @@
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.6
-  - 2.2.2
+  - 2.1.10
+  - 2.2.5
+  - 2.3.1
   - ruby-head
 
 script: "bundle exec rake spec"

data/ChangeLog

@@ -1,3 +1,11 @@
+== 2016-07-07 version 0.8.48
+
+* Drop 2.0.0 support #33
+* Show taken time of delete and acquire (#33)
+* To ease the migration delete v0.8.44 finished tasks too (#34)
+* Add stress test tool (#35)
+* Fix cleanup interval count (#36)
+
 == 2016-04-15 version 0.8.47
 
 * require libraries before fork to help CoW

data/README.md

@@ -189,6 +189,14 @@ additional configuration:
 
 Not implemented yet.
 
+## config/perfectqueue.yml Example
+
+```
+development:
+  type: rdb_compat
+  url: mysql2://root:@localhost:3306/perfectqueue
+  table: queues
+```
 
 ## Command line management tool
 
@@ -241,7 +249,7 @@ options for run:
 
 ### force finish a tasks
 
-    $ perfectqueue cancel_request k2
+    $ perfectqueue force_finish k2
 
 ### running a worker
 

data/bin/stress

@@ -0,0 +1,201 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'sequel'
+require 'perfectqueue'
+require 'perfectqueue/backend/rdb_compat'
+require 'securerandom'
+require 'optparse'
+
+queue_config =
+  {
+  :type => 'rdb_compat',
+  :url => "mysql2://testuser:@localhost:3306/perfectqueue_test",
+  :table => 'test_stress',
+  :disable_resource_limit => true,  # TODO backend-specific test cases
+  :cleanup_interval => 200,
+  insert_processes: 0,
+  worker_processes: 0,
+}
+opt = OptionParser.new
+
+opt.on('--url URL', 'database url') {|v| queue_config[:url] = v }
+opt.on('--table TABLE', 'table name') {|v| queue_config[:table] = v }
+opt.on('--disable_resource_limit=true', TrueClass, 'use resource limit or not') {|v| queue_config[:disable_resource_limit] = v }
+opt.on('--cleanup_interval SECOND', Integer, 'cleanup interval') {|v| queue_config[:cleanup_interval] = v }
+opt.on('--insert_processes NUM', Integer, 'inserts') {|v| queue_config[:insert_processes] = v }
+opt.on('--worker_processes NUM', Integer, 'workers') {|v| queue_config[:worker_processes] = v }
+opt.on('--retention-time SECOND', Integer, 'retention time') {|v| queue_config[:retention_time] = v }
+opt.parse!(ARGV)
+
+module PerfectQueue
+  class Queue
+    def submit1000(data)
+      @client.submit1000(data)
+    end
+  end
+  class Client
+    def submit1000(data)
+      @backend.submit1000(data)
+    end
+  end
+  module Backend
+    class RDBCompatBackend
+      def submit1000(h)
+        rd = Random.new
+        i = 0
+        connect {
+          begin
+            begin
+              n = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+              submit0("import.1/main.action_#{n}_#{rd.hex(80)}", 'user02', h, now: n)
+            end while (i+=1) < 10000
+          end
+        }
+      end
+
+      # => Task
+      def submit0(key, type, data, options)
+        now = (options[:now] || Time.now).to_i
+        now = 1 if now < 1  # 0 means cancel requested
+        run_at = (options[:run_at] || now).to_i
+        user = options[:user]
+        user = user.to_s if user
+        max_running = options[:max_running]
+        data = data ? data.dup : {}
+        data['type'] = type
+
+        d = data.to_json
+
+        if options[:compression] == 'gzip'
+          require 'zlib'
+          require 'stringio'
+          io = StringIO.new
+          gz = Zlib::GzipWriter.new(io)
+          begin
+            gz.write(d)
+          ensure
+            gz.close
+          end
+          d = io.string
+          d.force_encoding('ASCII-8BIT') if d.respond_to?(:force_encoding)
+          d = Sequel::SQL::Blob.new(d)
+        end
+
+        @db[
+          "INSERT INTO `#{@table}` (id, timeout, data, created_at, resource, max_running) VALUES (?, ?, ?, ?, ?, ?)",
+          key, run_at, d, now, user, max_running
+        ].insert
+      end
+    end
+  end
+end
+
+
+def insert1000(queue)
+  Process.setproctitle'bin/stress/insert'
+  while 1
+    t0 = t = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+    h = {"path":"in/1/main.action.e9f070b5bfea96442af13ce6acc36699_0f7ad8aee859867aae303190e372ec1e.msgpack.gz",
+"size":11373,
+"format":"msgpack.gz",
+"account_id":1,
+"start_at":nil,
+"database":"main",
+"table":"action",
+"table_path":"action_20160516_d85364_e43b23f4",
+"table_id":123456,
+"user":1,
+"c3769c43b07202bf6f456f4b99639a15":"1/main/action_20160516_d85364_e43b23f4",
+"hogefuga":"e9f070b5bfea96442af13ce6acc36699",
+"unique_id":"2d7325ef94e3eda8a20b65d33696d45a",
+"hoge_callback":{"url":"f117c9682a9c5129275f39493599272f809fcf3064edbfacc1c0a5",
+"headers":{"21a166dedd97bc":"TD1 4f8e44c0880086628ad7349e69d7d6ba19ce05ae"},
+"params":{}},
+"params":{}}
+    begin
+      queue.submit1000(h)
+    rescue Sequel::DatabaseError
+      p $!
+      sleep 5
+    end
+    t = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+    puts "#{__method__}#{Process.pid}: #{t-t0}sec for 1000 inserts\n"
+  end
+rescue Interrupt
+  exit
+end
+
+def insert(queue)
+  while 1
+    rd = Random.new
+    i = 0
+    t0 = t = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+    h = {"path":"in/1/main.action.e9f070b5bfea96442af13ce6acc36699_0f7ad8aee859867aae303190e372ec1e.msgpack.gz",
+"size":11373,
+"format":"msgpack.gz",
+"account_id":1,
+"start_at":nil,
+"database":"main",
+"table":"action",
+"table_path":"action_20160516_d85364_e43b23f4",
+"table_id":123456,
+"user":1,
+"c3769c43b07202bf6f456f4b99639a15":"1/main/action_20160516_d85364_e43b23f4",
+"hogefuga":"e9f070b5bfea96442af13ce6acc36699",
+"unique_id":"2d7325ef94e3eda8a20b65d33696d45a",
+"hoge_callback":{"url":"f117c9682a9c5129275f39493599272f809fcf3064edbfacc1c0a5",
+"headers":{"21a166dedd97bc":"TD1 4f8e44c0880086628ad7349e69d7d6ba19ce05ae"},
+"params":{}},
+"params":{}}
+    begin
+      n = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+      queue.submit("import.1/main.action_#{n}_#{rd.hex(80)}", 'user02', h, now: t)
+    rescue
+      p $!
+      sleep 1
+    end while (i+=1) < 1000
+    t = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+    puts "#{__method__}#{Process.pid}: #{t-t0}sec for 1000 inserts\n"
+  end
+end
+
+def worker(queue)
+  while 1
+    i = 0
+    t0 = t = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+    begin
+      ary = queue.poll_multi(max_acquire: 11, now: t)
+      ary.each do |x|
+        x.finish!({})
+      end if ary
+      t = Process.clock_gettime(Process::CLOCK_REALTIME, :second)
+    rescue
+      p $!.class
+      sleep 1
+    end while (i+=1) < 100
+    puts "#{__method__}#{Process.pid}: #{t-t0}sec for 100 acquires\n"
+  end
+rescue Interrupt
+  exit
+end
+
+pids = []
+queue = PerfectQueue.open(queue_config)
+#queue.client.init_database(:force => true)
+queue.config[:insert_processes].times do
+  pids << fork { insert1000(queue) }
+end
+queue.config[:worker_processes].times do
+  #queue.client.backend.instance_variable_set(:@cleanup_interval_count, rand(200))
+  pids << fork { worker(queue) }
+end
+queue.close
+
+trap (:INT) do
+  pids.each do |pid|
+    Process.kill(:INT, pid) rescue nil
+    Process.waitpid(pid) rescue nil
+  end
+  exit
+end
+sleep while 1

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -105,7 +105,10 @@ SQL
         @prefetch_break_types = config[:prefetch_break_types] || []
 
         @cleanup_interval = config[:cleanup_interval] || DEFAULT_DELETE_INTERVAL
-        @cleanup_interval_count = 0
+        # If cleanup_interval > max_request_per_child / max_acquire,
+        # some processes won't run DELETE query.
+        # (it's not an issue when there are enough workers)
+        @cleanup_interval_count = @cleanup_interval > 0 ? rand(@cleanup_interval) : 0
       end
 
       attr_reader :db
@@ -212,18 +215,22 @@ SQL
       def acquire(alive_time, max_acquire, options)
         now = (options[:now] || Time.now).to_i
         next_timeout = now + alive_time
-
-        tasks = []
+        tasks = nil
+        t0 = nil
 
         if @cleanup_interval_count <= 0
           delete_timeout = now - DELETE_OFFSET
-          connect { # TODO: HERE should be still connect_locked ?
+          connect_locked {
+            t0=Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            @db["DELETE FROM `#{@table}` WHERE #{EVENT_HORIZON} < timeout && timeout <= ? AND created_at IS NULL", now].delete
             @db["DELETE FROM `#{@table}` WHERE timeout <= ? AND created_at IS NULL", delete_timeout].delete
             @cleanup_interval_count = @cleanup_interval
+            STDERR.puts"PQ:delete from #{@table}:%6f sec" % [Process.clock_gettime(Process::CLOCK_MONOTONIC)-t0]
           }
         end
 
         connect_locked {
+          t0=Process.clock_gettime(Process::CLOCK_MONOTONIC)
           tasks = []
           @db.fetch(@sql, now, now, max_acquire) {|row|
             attributes = create_attributes(nil, row)
@@ -256,6 +263,8 @@ SQL
 
           return tasks
         }
+      ensure
+        STDERR.puts "PQ:acquire from #{@table}:%6f sec (%d tasks)" % [Process.clock_gettime(Process::CLOCK_MONOTONIC)-t0,tasks.size] if tasks
       end
 
       # => nil

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.47"
+  VERSION = "0.8.48"
 end

data/perfectqueue.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
+  gem.required_ruby_version = '>= 2.1'
   gem.add_dependency "sequel", "~> 3.48.0"
   gem.add_development_dependency "rake", "~> 0.9.2"
   gem.add_development_dependency "rspec", "~> 3.3.0"

data/spec/supervisor_spec.rb

@@ -182,7 +182,7 @@ describe Supervisor do
       queue.submit('task01', 'test', {'sleep'=>4})
       sleep 2
       Process.kill(:TERM, Process.pid)
-      expect(@thread.join(3)).to eq(@thread)
+      expect(@thread.join(5)).to eq(@thread)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.47
+  version: 0.8.48
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-15 00:00:00.000000000 Z
+date: 2016-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -84,6 +84,7 @@ description: Highly available distributed cron built on RDBMS
 email: frsyuki@gmail.com
 executables:
 - perfectqueue
+- stress
 extensions: []
 extra_rdoc_files: []
 files:
@@ -96,6 +97,7 @@ files:
 - README.md
 - Rakefile
 - bin/perfectqueue
+- bin/stress
 - circle.yml
 - lib/perfectqueue.rb
 - lib/perfectqueue/application.rb
@@ -163,7 +165,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -200,4 +202,3 @@ test_files:
 - spec/task_metadata_spec.rb
 - spec/task_monitor_spec.rb
 - spec/task_spec.rb
-has_rdoc: false