perfectqueue 0.8.49 → 0.8.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 61295f3241d66b1d9f27aac68294e32344b09611
4
- data.tar.gz: 36371652be7e64c3d5c1b79396c37ff8f5e5344c
3
+ metadata.gz: ed8791f086ecef1178ae6d001dc4addeef32a14f
4
+ data.tar.gz: 342eb4a9e95c7a5fdcc29c909f1f7a99c1e236ea
5
5
  SHA512:
6
- metadata.gz: 2461e220ce8b2f45308db2f377b0816718c4222bc70bcdefec5b6c6fe2df0b3938b7e5d5ff87ad2888b7f20133c91d132a333aa9c0578928a231cb9e8f72b312
7
- data.tar.gz: ba6542077acbf7e914c22a90afbcd54833d6e06e42781e8d62300b8d74cda1dfdf29eb5f7bfcb90f3788547828bc79c9cf61e74676127f29b879b2b00f8878d9
6
+ metadata.gz: 4ca28812f77cfad84f2f4198de80093b41b46ae26f8ebf0fc13fe8054c785e7e24f766baf64839af1a4c0fadc8f38073e5b1abd7c962ad97a32f859a2b5b237e
7
+ data.tar.gz: 13f80b70c03b3a43538f7f687869db8918eb5a319ce7fc2503963d21a0f4fd5fa3f3a1d0163f6e19424c0001061f2b049e80287261fd230a3628e5cae1fd4f76
@@ -0,0 +1,19 @@
1
+ version: 2.1
2
+
3
+ executors:
4
+ executor-circle:
5
+ docker:
6
+ - image: buildpack-deps:jessie
7
+ working_directory: /tmp
8
+
9
+ jobs:
10
+ build:
11
+ executor: executor-circle
12
+ steps:
13
+ - run: echo "It has used Travis instead of CircleCI."
14
+
15
+ workflows:
16
+ version: 2
17
+ build_and_test:
18
+ jobs:
19
+ - build
@@ -1,9 +1,13 @@
1
1
  rvm:
2
- - 2.1.10
3
- - 2.2.5
4
- - 2.3.1
2
+ - 2.3.8
3
+ - 2.4.6
4
+ - 2.5.5
5
+ - 2.6.3
5
6
  - ruby-head
6
7
 
8
+ services:
9
+ - mysql
10
+
7
11
  script: "bundle exec rake spec"
8
12
 
9
13
  before_script:
data/ChangeLog CHANGED
@@ -1,3 +1,24 @@
1
+ == 2020-09-10 version 0.8.54
2
+
3
+ * Enhance logging on inter-process communication (#68)
4
+
5
+ == 2019-10-10 version 0.8.53
6
+
7
+ * Extract max_retry_count as config parameter (#66)
8
+
9
+ == 2019-07-31 version 0.8.52
10
+
11
+ * Add sleep lock retrying with exponential-backoff (#64)
12
+
13
+ == 2016-10-24 version 0.8.51
14
+
15
+ * Retry on temporary DatabaseConnectionError (#59)
16
+
17
+ == 2016-08-18 version 0.8.50
18
+
19
+ * Decrease GET_LOCK timeout from 60 to 10 seconds
20
+ * Add 0.5 to 30 seconds of sleep between GET_LOCK retrying
21
+
1
22
  == 2016-08-02 version 0.8.49
2
23
 
3
24
  * Revert v0.8.44 migration path (#38)
@@ -4,7 +4,6 @@ require_relative 'rdb_compat'
4
4
 
5
5
  module PerfectQueue::Backend
6
6
  class RDBBackend
7
- MAX_RETRY = ::PerfectQueue::Backend::RDBCompatBackend::MAX_RETRY
8
7
  DELETE_OFFSET = ::PerfectQueue::Backend::RDBCompatBackend::DELETE_OFFSET
9
8
  class Token < Struct.new(:key)
10
9
  end
@@ -21,6 +20,9 @@ module PerfectQueue::Backend
21
20
  host: u.host,
22
21
  port: u.port ? u.port.to_i : 3306
23
22
  }
23
+ @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
24
+ @max_retry_count = config.fetch(:max_retry_count, 10)
25
+ options[:connect_timeout] = config.fetch(:connect_timeout, 3)
24
26
  options[:sslca] = config[:sslca] if config[:sslca]
25
27
  db_name = u.path.split('/')[1]
26
28
  @db = Sequel.mysql2(db_name, options)
@@ -32,6 +34,7 @@ module PerfectQueue::Backend
32
34
  end
33
35
 
34
36
  attr_reader :db
37
+ attr_reader :max_retry_count
35
38
 
36
39
  def submit(id, data, time=Process.clock_gettime(Process::CLOCK_REALTIME, :second), resource=nil, max_running=nil)
37
40
  connect {
@@ -54,23 +57,31 @@ module PerfectQueue::Backend
54
57
  end
55
58
 
56
59
  private
57
- def connect(&block)
60
+ def connect
61
+ tmax = Process.clock_gettime(Process::CLOCK_REALTIME, :second) + @pq_connect_timeout
58
62
  @mutex.synchronize do
59
63
  retry_count = 0
60
64
  begin
61
- block.call
65
+ yield
66
+ rescue Sequel::DatabaseConnectionError
67
+ if (retry_count += 1) < @max_retry_count && tmax > Process.clock_gettime(Process::CLOCK_REALTIME, :second)
68
+ STDERR.puts "#{$!}\n retrying."
69
+ sleep 2
70
+ retry
71
+ end
72
+ STDERR.puts "#{$!}\n abort."
73
+ raise
62
74
  rescue
63
75
  # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
64
76
  if $!.to_s.include?('try restarting transaction')
65
- err = ([$!] + $!.backtrace.map {|bt| " #{bt}" }).join("\n")
77
+ err = $!.backtrace.map{|bt| " #{bt}" }.unshift($!).join("\n")
66
78
  retry_count += 1
67
- if retry_count < MAX_RETRY
68
- STDERR.puts err + "\n retrying."
79
+ if retry_count < @max_retry_count
80
+ STDERR.puts "#{err}\n retrying."
69
81
  sleep 0.5
70
82
  retry
71
- else
72
- STDERR.puts err + "\n abort."
73
83
  end
84
+ STDERR.puts "#{err}\n abort."
74
85
  end
75
86
  raise
76
87
  ensure
@@ -33,12 +33,17 @@ module PerfectQueue
33
33
  DELETE_OFFSET = 10_0000_0000
34
34
  EVENT_HORIZON = 13_0000_0000 # 2011-03-13 07:06:40 UTC
35
35
 
36
+ LOCK_RETRY_INITIAL_INTERVAL = 0.5
37
+ LOCK_RETRY_MAX_INTERVAL = 30
38
+
36
39
  class Token < Struct.new(:key)
37
40
  end
38
41
 
39
42
  def initialize(client, config)
40
43
  super
41
44
 
45
+ @pq_connect_timeout = config.fetch(:pq_connect_timeout, 20)
46
+ @max_retry_count = config.fetch(:max_retry_count, 10)
42
47
  url = config[:url]
43
48
  @table = config[:table]
44
49
  unless @table
@@ -46,7 +51,9 @@ module PerfectQueue
46
51
  end
47
52
 
48
53
  if /\Amysql2:/i =~ url
49
- @db = Sequel.connect(url, {max_connections: 1, sslca: config[:sslca]})
54
+ options = {max_connections: 1, sslca: config[:sslca]}
55
+ options[:connect_timeout] = config.fetch(:connect_timeout, 3)
56
+ @db = Sequel.connect(url, options)
50
57
  if config.fetch(:use_connection_pooling, nil) != nil
51
58
  @use_connection_pooling = !!config[:use_connection_pooling]
52
59
  else
@@ -54,11 +61,14 @@ module PerfectQueue
54
61
  end
55
62
  @table_lock = lambda {
56
63
  locked = nil
64
+ interval = LOCK_RETRY_INITIAL_INTERVAL
57
65
  loop do
58
66
  @db.fetch("SELECT GET_LOCK('#{@table}', #{LOCK_WAIT_TIMEOUT}) locked") do |row|
59
67
  locked = true if row[:locked] == 1
60
68
  end
61
69
  break if locked
70
+ sleep interval
71
+ interval = [interval * 2, LOCK_RETRY_MAX_INTERVAL].min
62
72
  end
63
73
  }
64
74
  @table_unlock = lambda {
@@ -112,10 +122,10 @@ SQL
112
122
  end
113
123
 
114
124
  attr_reader :db
125
+ attr_reader :max_retry_count
115
126
 
116
127
  KEEPALIVE = 10
117
- MAX_RETRY = 10
118
- LOCK_WAIT_TIMEOUT = 60
128
+ LOCK_WAIT_TIMEOUT = 10
119
129
  DEFAULT_DELETE_INTERVAL = 20
120
130
 
121
131
  def init_database(options)
@@ -333,7 +343,7 @@ SQL
333
343
  end
334
344
 
335
345
  protected
336
- def connect_locked(&block)
346
+ def connect_locked
337
347
  connect {
338
348
  locked = false
339
349
 
@@ -343,7 +353,7 @@ SQL
343
353
  locked = true
344
354
  end
345
355
 
346
- return block.call
356
+ return yield
347
357
  ensure
348
358
  if @use_connection_pooling && locked
349
359
  @table_unlock.call
@@ -352,22 +362,31 @@ SQL
352
362
  }
353
363
  end
354
364
 
355
- def connect(&block)
365
+ def connect
356
366
  now = Time.now.to_i
367
+ tmax = now + @pq_connect_timeout
357
368
  @mutex.synchronize do
358
369
  # keepalive_timeout
359
370
  @db.disconnect if now - @last_time > KEEPALIVE
360
371
 
361
372
  count = 0
362
373
  begin
363
- block.call
374
+ yield
364
375
  @last_time = now
376
+ rescue Sequel::DatabaseConnectionError
377
+ if (count += 1) < @max_retry_count && tmax > Time.now.to_i
378
+ STDERR.puts "#{$!}\n retrying."
379
+ sleep 2
380
+ retry
381
+ end
382
+ STDERR.puts "#{$!}\n abort."
383
+ raise
365
384
  rescue
366
385
  # workaround for "Mysql2::Error: Deadlock found when trying to get lock; try restarting transaction" error
367
386
  if $!.to_s.include?('try restarting transaction')
368
387
  err = ([$!] + $!.backtrace.map {|bt| " #{bt}" }).join("\n")
369
388
  count += 1
370
- if count < MAX_RETRY
389
+ if count < @max_retry_count
371
390
  STDERR.puts err + "\n retrying."
372
391
  sleep rand
373
392
  retry
@@ -102,8 +102,8 @@ module PerfectQueue
102
102
  def send_signal(sig)
103
103
  begin
104
104
  Process.kill(sig, @pid)
105
- rescue Errno::ESRCH, Errno::EPERM
106
- # TODO log?
105
+ rescue Errno::ESRCH, Errno::EPERM => e
106
+ @log.info "#{e.class}: #{e.message}\n#{e.backtrace}"
107
107
  end
108
108
  end
109
109
 
@@ -121,7 +121,7 @@ module PerfectQueue
121
121
  end
122
122
 
123
123
  def process(task)
124
- @log.info "acquired task task=#{task.key} id=#{@processor_id}: #{task.inspect}"
124
+ @log.info "acquired task task=#{task.key} id=#{@processor_id}"
125
125
  begin
126
126
  r = @runner.new(task)
127
127
  @tm.set_task(task, r)
@@ -1,3 +1,3 @@
1
1
  module PerfectQueue
2
- VERSION = "0.8.49"
2
+ VERSION = "0.8.54"
3
3
  end
@@ -28,6 +28,7 @@ module PerfectQueue
28
28
  config = block.call
29
29
 
30
30
  @config = config
31
+ @log = config[:logger] || Logger.new(STDERR)
31
32
  @runner = runner
32
33
 
33
34
  @detach_wait = config[:detach_wait] || config['detach_wait'] || 10.0
@@ -66,9 +67,11 @@ module PerfectQueue
66
67
 
67
68
  else
68
69
  # child process finished unexpectedly
70
+ @log.info "Child process finished unexpectedly pid=#{pid}"
69
71
  end
70
72
 
71
- rescue Errno::ECHILD
73
+ rescue Errno::ECHILD => e
74
+ @log.info "#{e.class}: #{e.message}\n#{e.backtrace}"
72
75
  end
73
76
  end
74
77
 
@@ -130,4 +133,3 @@ module PerfectQueue
130
133
  end
131
134
 
132
135
  end
133
-
@@ -55,7 +55,7 @@ describe Backend::RDBBackend do
55
55
  end
56
56
  context 'error' do
57
57
  it 'returns block result' do
58
- expect(RuntimeError).to receive(:new).exactly(Backend::RDBBackend::MAX_RETRY).and_call_original
58
+ expect(RuntimeError).to receive(:new).exactly(db.max_retry_count).and_call_original
59
59
  allow(STDERR).to receive(:puts)
60
60
  allow(db).to receive(:sleep)
61
61
  expect do
@@ -65,5 +65,19 @@ describe Backend::RDBBackend do
65
65
  end.to raise_error(RuntimeError)
66
66
  end
67
67
  end
68
+ context 'cannot connect' do
69
+ let (:uri){ 'mysql2://root:@nonexistent/perfectqueue_test' }
70
+ let (:db) do
71
+ Backend::RDBBackend.new(uri, table)
72
+ end
73
+ it 'raises Sequel::DatabaseConnectionError' do
74
+ allow(STDERR).to receive(:puts)
75
+ slept = 0
76
+ expect(db).to receive(:sleep).exactly(9).times{|n| slept += n }
77
+ expect(db.db).to receive(:connect).exactly(10).times.and_call_original
78
+ expect{ db.__send__(:connect){ db.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
79
+ expect(slept).to be < 30
80
+ end
81
+ end
68
82
  end
69
83
  end
@@ -367,7 +367,7 @@ describe Backend::RDBCompatBackend do
367
367
  end
368
368
  context 'error' do
369
369
  it 'returns block result' do
370
- expect(RuntimeError).to receive(:new).exactly(Backend::RDBCompatBackend::MAX_RETRY).and_call_original
370
+ expect(RuntimeError).to receive(:new).exactly(db.max_retry_count).and_call_original
371
371
  allow(STDERR).to receive(:puts)
372
372
  allow(db).to receive(:sleep)
373
373
  expect do
@@ -376,6 +376,18 @@ describe Backend::RDBCompatBackend do
376
376
  end
377
377
  end.to raise_error(RuntimeError)
378
378
  end
379
+ context 'cannot connect' do
380
+ let (:config){ {url: 'mysql2://root:@nonexistent/perfectqueue_test', table: table} }
381
+ it 'raises Sequel::DatabaseConnectionError' do
382
+ allow(STDERR).to receive(:puts)
383
+ d = Backend::RDBCompatBackend.new(client, config)
384
+ slept = 0
385
+ expect(d).to receive(:sleep).exactly(9).times{|n| slept += n }
386
+ expect(d.db).to receive(:connect).exactly(10).times.and_call_original
387
+ expect{ d.__send__(:connect){ d.db.run('SELECT 1;') } }.to raise_error(Sequel::DatabaseConnectionError)
388
+ expect(slept).to eq(18)
389
+ end
390
+ end
379
391
  end
380
392
  end
381
393
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: perfectqueue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.49
4
+ version: 0.8.54
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-08-02 00:00:00.000000000 Z
11
+ date: 2020-09-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sequel
@@ -88,6 +88,7 @@ executables:
88
88
  extensions: []
89
89
  extra_rdoc_files: []
90
90
  files:
91
+ - ".circleci/config.yml"
91
92
  - ".gitignore"
92
93
  - ".travis.yml"
93
94
  - ChangeLog
@@ -98,7 +99,6 @@ files:
98
99
  - Rakefile
99
100
  - bin/perfectqueue
100
101
  - bin/stress
101
- - circle.yml
102
102
  - lib/perfectqueue.rb
103
103
  - lib/perfectqueue/application.rb
104
104
  - lib/perfectqueue/application/base.rb
@@ -177,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
177
177
  version: '0'
178
178
  requirements: []
179
179
  rubyforge_project:
180
- rubygems_version: 2.5.1
180
+ rubygems_version: 2.5.2.3
181
181
  signing_key:
182
182
  specification_version: 4
183
183
  summary: Highly available distributed cron built on RDBMS
@@ -208,4 +208,3 @@ test_files:
208
208
  - spec/task_metadata_spec.rb
209
209
  - spec/task_monitor_spec.rb
210
210
  - spec/task_spec.rb
211
- has_rdoc: false
data/circle.yml DELETED
@@ -1,7 +0,0 @@
1
- machine:
2
- ruby:
3
- version: 2.2.2
4
-
5
- database:
6
- pre:
7
- - mysql -e 'create database perfectqueue_test;'