perfectqueue 0.8.48 → 0.8.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56f251e9261948702f9f058179343788c08ea08e
-  data.tar.gz: 4bd8336b27bfc2097a22581120e872fe621d4c53
+  metadata.gz: 61295f3241d66b1d9f27aac68294e32344b09611
+  data.tar.gz: 36371652be7e64c3d5c1b79396c37ff8f5e5344c
 SHA512:
-  metadata.gz: 0a826da88811b6f511b4e4b0a6bdab9255a0a792848874b761fb9c2b6014fb3ac32f1c2a8231c8bdd54dfec82bb37fbd46b87664d392f829fb9fadfcb4388d3e
-  data.tar.gz: 4591e5a7f253502dafb1687ce57d572fa8b7742b2482be6938d41ce501d9e42f44cc4a2ca3e9a13a4a7de6f50f3518831b2bc9ba60bbac5ec29930ea67c36602
+  metadata.gz: 2461e220ce8b2f45308db2f377b0816718c4222bc70bcdefec5b6c6fe2df0b3938b7e5d5ff87ad2888b7f20133c91d132a333aa9c0578928a231cb9e8f72b312
+  data.tar.gz: ba6542077acbf7e914c22a90afbcd54833d6e06e42781e8d62300b8d74cda1dfdf29eb5f7bfcb90f3788547828bc79c9cf61e74676127f29b879b2b00f8878d9

data/ChangeLog

@@ -1,3 +1,10 @@
+== 2016-08-02 version 0.8.49
+
+* Revert v0.8.44 migration path (#38)
+* v0.8.48's timeout express taks's life (#39)
+* Use Sequel::UniqueConstraintViolation (#40)
+* Merge v07 branch (#45)
+
 == 2016-07-07 version 0.8.48
 
 * Drop 2.0.0 support #33

data/README.md

@@ -4,7 +4,7 @@
 [![Coverage Status](https://coveralls.io/repos/treasure-data/perfectqueue/badge.svg?branch=master&service=github)](https://coveralls.io/github/treasure-data/perfectqueue?branch=master)
 
 PerfectQueue is a highly available distributed queue built on top of RDBMS.
-PerfectQueue provides similar API to Amazon SQS, while PerfectQueue focues on reliability and flexible scheduling rather than scalability.
+PerfectQueue provides similar API to Amazon SQS, while PerfectQueue focuses on reliability and flexible scheduling rather than scalability.
 
 PerfectQueue introduces following concepts:
 

data/lib/perfectqueue/backend/null.rb

@@ -0,0 +1,27 @@
+module PerfectQueue::Backend
+  class NullBackend
+    def list(&block)
+      nil
+    end
+
+    def acquire(timeout, now=Time.now.to_i)
+      nil
+    end
+
+    def finish(token, delete_timeout=3600, now=Time.now.to_i)
+      true
+    end
+
+    def update(token, timeout)
+      nil
+    end
+
+    def cancel(id, delete_timeout=3600, now=Time.now.to_i)
+      true
+    end
+
+    def submit(id, data, time=Time.now.to_i, resource=nil, max_running=nil)
+      true
+    end
+  end
+end

data/lib/perfectqueue/backend/rdb.rb

@@ -0,0 +1,82 @@
+require 'sequel'
+require 'uri'
+require_relative 'rdb_compat'
+
+module PerfectQueue::Backend
+  class RDBBackend
+    MAX_RETRY = ::PerfectQueue::Backend::RDBCompatBackend::MAX_RETRY
+    DELETE_OFFSET = ::PerfectQueue::Backend::RDBCompatBackend::DELETE_OFFSET
+    class Token < Struct.new(:key)
+    end
+
+    def initialize(uri, table, config={})
+      @uri = uri
+      @table = table
+
+      u = URI.parse(@uri)
+      options = {
+        max_connections: 1,
+        user: u.user,
+        password: u.password,
+        host: u.host,
+        port: u.port ? u.port.to_i : 3306
+      }
+      options[:sslca] = config[:sslca] if config[:sslca]
+      db_name = u.path.split('/')[1]
+      @db = Sequel.mysql2(db_name, options)
+
+      @mutex = Mutex.new
+      connect {
+        # connection test
+      }
+    end
+
+    attr_reader :db
+
+    def submit(id, data, time=Process.clock_gettime(Process::CLOCK_REALTIME, :second), resource=nil, max_running=nil)
+      connect {
+        begin
+          data = Sequel::SQL::Blob.new(data)
+          @db.sql_log_level = :debug
+          n = @db["INSERT INTO `#{@table}` (id, timeout, data, created_at, resource, max_running) VALUES (?, ?, ?, ?, ?, ?);", id, time, data, time, resource, max_running].insert
+          return true
+        rescue Sequel::UniqueConstraintViolation => e
+          return nil
+        end
+      }
+    end
+
+    def cancel(id, delete_timeout=3600, now=Process.clock_gettime(Process::CLOCK_REALTIME, :second))
+      connect {
+        n = @db["UPDATE `#{@table}` SET timeout=?, created_at=NULL, resource=NULL WHERE id=? AND created_at IS NOT NULL;", now+delete_timeout-DELETE_OFFSET, id].update
+        return n > 0
+      }
+    end
+
+    private
+    def connect(&block)
+      @mutex.synchronize do
+        retry_count = 0
+        begin
+          block.call
+        rescue
+          # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
+          if $!.to_s.include?('try restarting transaction')
+            err = ([$!] + $!.backtrace.map {|bt| "  #{bt}" }).join("\n")
+            retry_count += 1
+            if retry_count < MAX_RETRY
+              STDERR.puts err + "\n  retrying."
+              sleep 0.5
+              retry
+            else
+              STDERR.puts err + "\n  abort."
+            end
+          end
+          raise
+        ensure
+          @db.disconnect
+        end
+      end
+    end
+  end
+end

data/lib/perfectqueue/backend/rdb_compat.rb

@@ -205,7 +205,7 @@ SQL
               key, run_at, d, now, user, max_running
             ].insert
             return Task.new(@client, key)
-          rescue Sequel::DatabaseError
+          rescue Sequel::UniqueConstraintViolation
             raise IdempotentAlreadyExistsError, "task key=#{key} already exists"
           end
         }
@@ -219,11 +219,9 @@ SQL
         t0 = nil
 
         if @cleanup_interval_count <= 0
-          delete_timeout = now - DELETE_OFFSET
-          connect_locked {
+          connect { # TODO: HERE should be still connect_locked ?
             t0=Process.clock_gettime(Process::CLOCK_MONOTONIC)
-            @db["DELETE FROM `#{@table}` WHERE #{EVENT_HORIZON} < timeout && timeout <= ? AND created_at IS NULL", now].delete
-            @db["DELETE FROM `#{@table}` WHERE timeout <= ? AND created_at IS NULL", delete_timeout].delete
+            @db["DELETE FROM `#{@table}` WHERE timeout <= ?", now-DELETE_OFFSET].delete
             @cleanup_interval_count = @cleanup_interval
             STDERR.puts"PQ:delete from #{@table}:%6f sec" % [Process.clock_gettime(Process::CLOCK_MONOTONIC)-t0]
           }

data/lib/perfectqueue/version.rb

@@ -1,3 +1,3 @@
 module PerfectQueue
-  VERSION = "0.8.48"
+  VERSION = "0.8.49"
 end

data/spec/null_backend_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+require 'perfectqueue/backend/null'
+
+describe Backend::NullBackend do
+  let (:backend){ Backend::NullBackend.new }
+  describe '#list' do
+    subject { backend.list{} }
+     it { is_expected.to be_nil }
+  end
+  describe '#acquire' do
+    subject { backend.acquire(double('timeout')) }
+     it { is_expected.to be_nil }
+  end
+  describe '#finish' do
+    subject { backend.finish(double('token')) }
+     it { is_expected.to be true }
+  end
+  describe '#update' do
+    subject { backend.update(double('token'), double('timeout')) }
+     it { is_expected.to be_nil }
+  end
+  describe '#cancel' do
+    subject { backend.cancel(double('id')) }
+     it { is_expected.to be true }
+  end
+  describe '#submit' do
+    subject { backend.submit(double('id'), double('data')) }
+     it { is_expected.to be true }
+  end
+end

data/spec/rdb_backend_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+require 'perfectqueue/backend/rdb'
+
+describe Backend::RDBBackend do
+  let (:now){ Time.now.to_i }
+  let (:uri){ 'mysql2://root:@localhost/perfectqueue_test' }
+  let (:table){ 'test_queues' }
+  let (:db) do
+    d = Backend::RDBCompatBackend.new(double, url: uri, table: table)
+    s = d.db
+    s.tables.each{|t| s.drop_table(t) }
+    d.init_database({})
+    Backend::RDBBackend.new(uri, table)
+  end
+
+  context '.new' do
+    it 'supports mysql' do
+      expect(Backend::RDBBackend.new(uri, table)).to be_an_instance_of(Backend::RDBBackend)
+    end
+  end
+
+  context '#submit' do
+    it 'adds task' do
+      db.submit('key', '{"foo":"bar"}')
+      row = db.db.fetch("SELECT * FROM `#{table}` WHERE id=? LIMIT 1", 'key').first
+      expect(row[:created_at]).not_to be_nil
+      expect(row[:data]).to eq('{"foo":"bar"}')
+    end
+  end
+
+  context '#cancel' do
+    let (:key){ 'key' }
+    context 'have the task' do
+      before do
+        db.submit(key, '{}')
+      end
+      it 'returns true' do
+        expect(db.cancel(key)).to be true
+        row = db.db.fetch("SELECT created_at FROM `#{table}` WHERE id=? LIMIT 1", key).first
+        expect(row[:created_at]).to be_nil
+      end
+    end
+    context 'already canceled' do
+      it 'returns false' do
+        expect(db.cancel(key)).to be false
+      end
+    end
+  end
+
+  context '#connect' do
+    context 'normal' do
+      it 'returns nil' do
+        expect(db.__send__(:connect){ }).to be_nil
+      end
+    end
+    context 'error' do
+      it 'returns block result' do
+        expect(RuntimeError).to receive(:new).exactly(Backend::RDBBackend::MAX_RETRY).and_call_original
+        allow(STDERR).to receive(:puts)
+        allow(db).to receive(:sleep)
+        expect do
+          db.__send__(:connect) do
+            raise RuntimeError.new('try restarting transaction')
+          end
+        end.to raise_error(RuntimeError)
+      end
+    end
+  end
+end

data/spec/rdb_compat_backend_spec.rb

@@ -208,7 +208,7 @@ describe Backend::RDBCompatBackend do
         db.list({}) do |task|
           expect(task.timeout).to eq now0.to_time
         end
-        ary = db.acquire(alive_time, max_acquire, {})
+        ary = db.acquire(alive_time, max_acquire, {now: now})
         expect(ary).to be_an_instance_of(Array)
         expect(ary.size).to eq(3)
         expect(ary[0]).to be_an_instance_of(AcquiredTask)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perfectqueue
 version: !ruby/object:Gem::Version
-  version: 0.8.48
+  version: 0.8.49
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-07 00:00:00.000000000 Z
+date: 2016-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -106,6 +106,8 @@ files:
 - lib/perfectqueue/application/dispatch.rb
 - lib/perfectqueue/application/router.rb
 - lib/perfectqueue/backend.rb
+- lib/perfectqueue/backend/null.rb
+- lib/perfectqueue/backend/rdb.rb
 - lib/perfectqueue/backend/rdb_compat.rb
 - lib/perfectqueue/blocking_flag.rb
 - lib/perfectqueue/client.rb
@@ -142,7 +144,9 @@ files:
 - spec/multiprocess/child_process_spec.rb
 - spec/multiprocess/fork_processor_spec.rb
 - spec/multiprocess/thread_processor_spec.rb
+- spec/null_backend_spec.rb
 - spec/queue_spec.rb
+- spec/rdb_backend_spec.rb
 - spec/rdb_compat_backend_spec.rb
 - spec/rdb_stress.rb
 - spec/runner_spec.rb
@@ -173,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Highly available distributed cron built on RDBMS
@@ -191,7 +195,9 @@ test_files:
 - spec/multiprocess/child_process_spec.rb
 - spec/multiprocess/fork_processor_spec.rb
 - spec/multiprocess/thread_processor_spec.rb
+- spec/null_backend_spec.rb
 - spec/queue_spec.rb
+- spec/rdb_backend_spec.rb
 - spec/rdb_compat_backend_spec.rb
 - spec/rdb_stress.rb
 - spec/runner_spec.rb
@@ -202,3 +208,4 @@ test_files:
 - spec/task_metadata_spec.rb
 - spec/task_monitor_spec.rb
 - spec/task_spec.rb
+has_rdoc: false