liveqa 1.9.3 → 1.9.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENCE +2 -2
- data/lib/liveqa/processor/async.rb +14 -9
- data/lib/liveqa/processor/batch.rb +10 -7
- data/lib/liveqa/processor/worker.rb +82 -39
- data/lib/liveqa/version.rb +1 -1
- data/spec/lib/liveqa/batch_spec.rb +1 -1
- data/spec/lib/liveqa/processor/async_spec.rb +18 -1
- data/spec/lib/liveqa/processor/batch_spec.rb +6 -3
- data/spec/lib/liveqa/processor/worker_spec.rb +85 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 46d9bc855dce533eaf940cc977b28d8904fed773
|
4
|
+
data.tar.gz: eaf6c87c0118f7a48e3e699101653fb99c89cf9a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cb08f5e030859cd1c0b1939b0c3d325a1e35b77e916c2358d08135221c1265a047f7d396e322dd8662002b3e47d1c25d379e9c35631e21faed18b0edc4f6d26f
|
7
|
+
data.tar.gz: 5dce2d321778216a67beab91f2243ba167bbcb12612d0d3aec024d0952e81527320e50eeb50a7e95d06b666f45329f843c0bd59111709c1bc5e43e3cd764ccbc
|
data/LICENCE
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
The MIT License
|
2
2
|
|
3
|
-
Copyright (c) 2018- LiveQA, Inc. (https://www.liveqa.
|
3
|
+
Copyright (c) 2018- LiveQA, Inc. (https://www.liveqa.io)
|
4
4
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
18
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
19
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
20
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
-
SOFTWARE.
|
21
|
+
SOFTWARE.
|
@@ -3,29 +3,34 @@ module LiveQA
|
|
3
3
|
class Async
|
4
4
|
|
5
5
|
def initialize
|
6
|
-
@
|
7
|
-
@
|
8
|
-
@worker = Worker.new(@queue,
|
6
|
+
@queue = Queue.new
|
7
|
+
@state_worker = Concurrent::AtomicBoolean.new(true)
|
8
|
+
@worker = Worker.new(@queue, @state_worker)
|
9
9
|
|
10
10
|
at_exit do
|
11
|
-
if worker_running?
|
12
|
-
executor.shutdown
|
13
|
-
executor.wait_for_termination
|
14
|
-
end
|
11
|
+
shutdown_worker if worker_running?
|
15
12
|
end
|
16
13
|
end
|
17
14
|
|
18
15
|
def enqueue(attributes)
|
19
|
-
|
16
|
+
ensure_worker_running
|
20
17
|
|
21
18
|
@queue << attributes
|
22
|
-
ensure_worker_running
|
23
19
|
|
24
20
|
true
|
25
21
|
end
|
26
22
|
|
27
23
|
private
|
28
24
|
|
25
|
+
def shutdown_worker
|
26
|
+
@state_worker.make_false
|
27
|
+
@queue << LiveQA::Processor::Worker::SHUTDOWN_MESSAGE
|
28
|
+
@worker_thread.wait
|
29
|
+
|
30
|
+
executor.shutdown
|
31
|
+
executor.wait_for_termination
|
32
|
+
end
|
33
|
+
|
29
34
|
def executor
|
30
35
|
@executor ||= Concurrent.global_io_executor
|
31
36
|
end
|
@@ -8,6 +8,8 @@ module LiveQA
|
|
8
8
|
MAX_MESSAGES = 100
|
9
9
|
MAX_MESSAGE_BYTES = 32_768 # 32Kb
|
10
10
|
|
11
|
+
DEFAULT_RETRY_TIME = 60
|
12
|
+
|
11
13
|
MAX_RETRY = 10
|
12
14
|
RETRY_MAP = {
|
13
15
|
1 => 2,
|
@@ -43,19 +45,20 @@ module LiveQA
|
|
43
45
|
true
|
44
46
|
end
|
45
47
|
|
48
|
+
def empty?
|
49
|
+
@messages.size.zero?
|
50
|
+
end
|
51
|
+
|
46
52
|
def full?
|
47
53
|
max_messages_reached? || max_size_reached?
|
48
54
|
end
|
49
55
|
|
50
|
-
def
|
51
|
-
@retry_count
|
52
|
-
@retry_at = Time.now + (RETRY_MAP[@retry_count] || 120)
|
56
|
+
def next_retry
|
57
|
+
RETRY_MAP[@retry_count] || DEFAULT_RETRY_TIME
|
53
58
|
end
|
54
59
|
|
55
|
-
def
|
56
|
-
|
57
|
-
|
58
|
-
Time.now >= @retry_at
|
60
|
+
def update_retry
|
61
|
+
@retry_count += 1
|
59
62
|
end
|
60
63
|
|
61
64
|
def can_retry?
|
@@ -2,67 +2,110 @@ module LiveQA
|
|
2
2
|
module Processor
|
3
3
|
class Worker
|
4
4
|
|
5
|
-
|
5
|
+
FLUSH_INTERVAL_SECONDS = 3
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
attr_reader :executor
|
7
|
+
FLUSH_MESSAGE = Object.new
|
8
|
+
SHUTDOWN_MESSAGE = Object.new
|
10
9
|
|
11
|
-
def initialize(queue,
|
10
|
+
def initialize(queue, state)
|
12
11
|
@queue = queue
|
13
|
-
@
|
12
|
+
@state = state
|
14
13
|
|
15
|
-
@
|
14
|
+
@batch = Batch.new
|
15
|
+
@promises = Concurrent::Array.new
|
16
|
+
|
17
|
+
@timer = Concurrent::TimerTask.new(execution_interval: FLUSH_INTERVAL_SECONDS) { @queue << FLUSH_MESSAGE }
|
18
|
+
@timer.execute
|
16
19
|
end
|
17
20
|
|
18
21
|
def run
|
19
|
-
while
|
20
|
-
|
21
|
-
|
22
|
-
sleep(DEFAULT_WAIT_TIME) if queue.empty?
|
22
|
+
while thread_active?
|
23
|
+
message = @queue.pop
|
23
24
|
|
24
|
-
|
25
|
-
|
26
|
-
send_batches
|
25
|
+
add_message_to_batch(message)
|
27
26
|
end
|
27
|
+
|
28
|
+
shutdown_worker
|
28
29
|
end
|
29
30
|
|
30
31
|
private
|
31
32
|
|
32
|
-
|
33
|
-
|
34
|
-
|
33
|
+
##
|
34
|
+
# Add the message to a batch
|
35
|
+
#
|
36
|
+
# - Add message to a batch if it's not a flush message
|
37
|
+
# - flush batch if flush message is receive or batch is full
|
38
|
+
def add_message_to_batch(message)
|
39
|
+
@batch << message if message != FLUSH_MESSAGE
|
40
|
+
flush if message == FLUSH_MESSAGE || @batch.full?
|
35
41
|
end
|
36
42
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
43
|
+
##
|
44
|
+
# Shutdown the worker
|
45
|
+
#
|
46
|
+
# - Close the timer
|
47
|
+
# - Retreive the last messages
|
48
|
+
# - Wait all the request to be completed
|
49
|
+
#
|
50
|
+
# rubocop:disable Lint/HandleExceptions
|
51
|
+
def shutdown_worker
|
52
|
+
@timer.shutdown
|
53
|
+
|
54
|
+
begin
|
55
|
+
until (message = @queue.pop(true)).nil?
|
56
|
+
add_message_to_batch(message)
|
57
|
+
end
|
58
|
+
rescue ThreadError => _
|
59
|
+
end
|
60
|
+
|
61
|
+
flush
|
62
|
+
|
63
|
+
@promises.each do |promise|
|
64
|
+
promise.wait if promise && !promise.fulfilled?
|
65
|
+
end
|
41
66
|
end
|
67
|
+
# rubocop:enable Lint/HandleExceptions
|
42
68
|
|
43
|
-
def
|
44
|
-
|
45
|
-
batch = batches.pop
|
69
|
+
def flush
|
70
|
+
return if @batch.empty?
|
46
71
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
end
|
72
|
+
send_batch(@batch)
|
73
|
+
@batch = Batch.new
|
74
|
+
end
|
51
75
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
76
|
+
# rubocop:disable Metrics/AbcSize
|
77
|
+
def send_batch(batch)
|
78
|
+
promise =
|
79
|
+
Concurrent::Promise
|
80
|
+
.new { LiveQA::Batch.create(Message.base.merge(data: batch.messages)) }
|
81
|
+
.on_success { @promises.delete(promise) }
|
82
|
+
.rescue do |error|
|
83
|
+
batch.update_retry
|
84
|
+
|
85
|
+
if error.is_a?(LiveQA::RequestError) && [401, 404].include?(error.http_status)
|
86
|
+
return logger.error(error.message)
|
87
|
+
end
|
88
|
+
|
89
|
+
if thread_active? && batch.can_retry?
|
90
|
+
Concurrent::ScheduledTask.new(batch.next_retry) { send_batch(batch) }.execute
|
91
|
+
else
|
92
|
+
@promises.delete(promise)
|
93
|
+
end
|
59
94
|
end
|
60
|
-
|
95
|
+
|
96
|
+
promise.execute
|
97
|
+
@promises << promise
|
98
|
+
rescue Concurrent::RejectedExecutionError => _
|
99
|
+
logger.error('Impossible to start a thread in closing application')
|
100
|
+
end
|
101
|
+
# rubocop:enable Metrics/AbcSize
|
102
|
+
|
103
|
+
def thread_active?
|
104
|
+
@state.true?
|
61
105
|
end
|
62
106
|
|
63
|
-
def
|
64
|
-
|
65
|
-
batches.push(batch) if batch.can_retry?
|
107
|
+
def logger
|
108
|
+
LiveQA.configurations.logger
|
66
109
|
end
|
67
110
|
|
68
111
|
end
|
data/lib/liveqa/version.rb
CHANGED
@@ -3,7 +3,7 @@ require 'spec_helper'
|
|
3
3
|
describe LiveQA::Batch do
|
4
4
|
|
5
5
|
describe '.create' do
|
6
|
-
let(:response) { double('LiveQA::Request', body: "{\"object\":\"
|
6
|
+
let(:response) { double('LiveQA::Request', body: "{\"object\":\"batch\",\"id\":41}") }
|
7
7
|
before { allow(LiveQA::Request).to receive(:execute).and_return(response) }
|
8
8
|
|
9
9
|
subject(:create) { LiveQA::Batch.create(data: []) }
|
@@ -5,7 +5,6 @@ describe LiveQA::Processor::Async do
|
|
5
5
|
let(:processor) { LiveQA::Processor::Async.new }
|
6
6
|
|
7
7
|
context 'enqueue a message' do
|
8
|
-
|
9
8
|
before do
|
10
9
|
allow(processor).to receive(:ensure_worker_running)
|
11
10
|
processor.enqueue({ test: 'hello' })
|
@@ -15,5 +14,23 @@ describe LiveQA::Processor::Async do
|
|
15
14
|
it { expect(processor.instance_variable_get(:@queue).length).to eq(1) }
|
16
15
|
end
|
17
16
|
|
17
|
+
context 'shutting down worker' do
|
18
|
+
before do
|
19
|
+
allow_message_expectations_on_nil
|
20
|
+
|
21
|
+
allow(processor.instance_variable_get(:@state_worker)).to receive(:make_false)
|
22
|
+
allow(processor.instance_variable_get(:@worker_thread)).to receive(:wait)
|
23
|
+
allow(Concurrent.global_io_executor).to receive(:shutdown)
|
24
|
+
allow(Concurrent.global_io_executor).to receive(:wait_for_termination)
|
25
|
+
|
26
|
+
processor.send(:shutdown_worker)
|
27
|
+
end
|
28
|
+
|
29
|
+
it { expect(processor.instance_variable_get(:@state_worker)).to have_received(:make_false) }
|
30
|
+
it { expect(processor.instance_variable_get(:@worker_thread)).to have_received(:wait) }
|
31
|
+
it { expect(Concurrent.global_io_executor).to have_received(:shutdown) }
|
32
|
+
it { expect(Concurrent.global_io_executor).to have_received(:wait_for_termination) }
|
33
|
+
it { expect(processor.instance_variable_get(:@queue).length).to eq(1) }
|
34
|
+
end
|
18
35
|
|
19
36
|
end
|
@@ -5,9 +5,9 @@ describe LiveQA::Processor::Batch do
|
|
5
5
|
|
6
6
|
context 'set defaults' do
|
7
7
|
it { expect(batch.messages).to eq([]) }
|
8
|
-
it { expect(batch.can_run?).to be_truthy }
|
9
8
|
it { expect(batch.can_retry?).to be_truthy }
|
10
9
|
it { expect(batch.full?).to be_falsey }
|
10
|
+
it { expect(batch.empty?).to be_truthy }
|
11
11
|
end
|
12
12
|
|
13
13
|
context 'add message' do
|
@@ -15,13 +15,14 @@ describe LiveQA::Processor::Batch do
|
|
15
15
|
|
16
16
|
it { expect(batch.messages).to eq([{ test: 'hello' }]) }
|
17
17
|
it { expect(batch.full?).to be_falsey }
|
18
|
+
it { expect(batch.empty?).to be_falsey }
|
18
19
|
end
|
19
20
|
|
20
21
|
context 'with extended task' do
|
21
22
|
before { batch.update_retry }
|
22
23
|
|
23
|
-
it { expect(batch.can_run?).to be_falsey }
|
24
24
|
it { expect(batch.can_retry?).to be_truthy }
|
25
|
+
it { expect(batch.next_retry).to eq(2) }
|
25
26
|
end
|
26
27
|
|
27
28
|
context 'with extended task' do
|
@@ -29,20 +30,22 @@ describe LiveQA::Processor::Batch do
|
|
29
30
|
11.times { batch.update_retry }
|
30
31
|
end
|
31
32
|
|
32
|
-
it { expect(batch.can_run?).to be_falsey }
|
33
33
|
it { expect(batch.can_retry?).to be_falsey }
|
34
|
+
it { expect(batch.next_retry).to eq(60) }
|
34
35
|
end
|
35
36
|
|
36
37
|
context 'with max number of messages reached' do
|
37
38
|
before { 100.times { batch << { test: 'hello' }}}
|
38
39
|
|
39
40
|
it { expect(batch.full?).to be_truthy }
|
41
|
+
it { expect(batch.empty?).to be_falsey }
|
40
42
|
end
|
41
43
|
|
42
44
|
context 'with max size reached' do
|
43
45
|
before { 20.times { batch << 1000.times.map {{ test: 'hello' }}}}
|
44
46
|
|
45
47
|
it { expect(batch.full?).to be_truthy }
|
48
|
+
it { expect(batch.empty?).to be_falsey }
|
46
49
|
end
|
47
50
|
|
48
51
|
end
|
@@ -1,25 +1,100 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe LiveQA::Processor::Worker do
|
4
|
+
let(:state_worker) { Concurrent::AtomicBoolean.new(true) }
|
4
5
|
let(:queue) { Queue.new }
|
5
|
-
let(:worker) { LiveQA::Processor::Worker.new(queue,
|
6
|
+
let(:worker) { LiveQA::Processor::Worker.new(queue, state_worker) }
|
7
|
+
let!(:current_batch) { worker.instance_variable_get(:@batch) }
|
6
8
|
|
7
|
-
|
9
|
+
describe 'private#add_message_to_batch' do
|
10
|
+
context 'add to batch' do
|
11
|
+
let(:message) { Object.new }
|
12
|
+
before { worker.send(:add_message_to_batch, message) }
|
13
|
+
|
14
|
+
it { expect(current_batch.messages).to eq([message]) }
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'with batch full' do
|
18
|
+
before do
|
19
|
+
allow(current_batch).to receive(:full?).and_return(true)
|
20
|
+
allow(worker).to receive(:flush)
|
21
|
+
worker.send(:add_message_to_batch, Object.new)
|
22
|
+
end
|
23
|
+
|
24
|
+
it { expect(worker).to have_received(:flush) }
|
25
|
+
end
|
26
|
+
|
27
|
+
context 'call flush' do
|
28
|
+
before do
|
29
|
+
allow(worker).to receive(:flush)
|
30
|
+
worker.send(:add_message_to_batch, LiveQA::Processor::Worker::FLUSH_MESSAGE)
|
31
|
+
end
|
32
|
+
|
33
|
+
it { expect(worker).to have_received(:flush) }
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe 'private#flush' do
|
8
38
|
before do
|
9
|
-
allow(
|
10
|
-
worker.
|
39
|
+
allow(current_batch).to receive(:empty?).and_return(batch_status)
|
40
|
+
allow(worker).to receive(:send_batch)
|
41
|
+
worker.send(:flush)
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'with not empty batch' do
|
45
|
+
let(:batch_status) { false }
|
46
|
+
|
47
|
+
it { expect(worker).to have_received(:send_batch).with(current_batch) }
|
48
|
+
it { expect(current_batch).to_not eq(worker.instance_variable_get(:@batch)) }
|
11
49
|
end
|
12
50
|
|
13
|
-
|
51
|
+
context 'with empty batch' do
|
52
|
+
let(:batch_status) { true }
|
53
|
+
|
54
|
+
it { expect(worker).to_not have_received(:send_batch) }
|
55
|
+
it { expect(current_batch).to eq(worker.instance_variable_get(:@batch)) }
|
56
|
+
end
|
14
57
|
end
|
15
58
|
|
16
|
-
|
59
|
+
describe 'private#send_batch' do
|
60
|
+
let(:batch) { LiveQA::Processor::Batch.new }
|
61
|
+
let(:batch_can_retry) { true }
|
62
|
+
|
17
63
|
before do
|
18
|
-
|
19
|
-
allow(
|
20
|
-
|
64
|
+
allow(LiveQA::Request).to receive(:execute).and_return(response)
|
65
|
+
allow(batch).to receive(:update_retry)
|
66
|
+
allow(batch).to receive(:can_retry?).and_return(batch_can_retry)
|
67
|
+
allow(Concurrent::ScheduledTask).to receive(:new)
|
68
|
+
worker.send(:send_batch, batch)
|
69
|
+
sleep(0.1)
|
70
|
+
end
|
71
|
+
|
72
|
+
context 'with valid request' do
|
73
|
+
let(:response) { double('LiveQA::Request', body: "{\"object\":\"batch\",\"id\":41}") }
|
74
|
+
|
75
|
+
it { expect(LiveQA::Request).to have_received(:execute) }
|
76
|
+
it { expect(batch).to_not have_received(:update_retry) }
|
77
|
+
it { expect(worker.instance_variable_get(:@promises).size).to eq(0) }
|
21
78
|
end
|
22
79
|
|
23
|
-
|
80
|
+
context 'with invalid request' do
|
81
|
+
let(:response) { nil }
|
82
|
+
|
83
|
+
it { expect(LiveQA::Request).to have_received(:execute) }
|
84
|
+
it { expect(batch).to have_received(:update_retry) }
|
85
|
+
it { expect(Concurrent::ScheduledTask).to have_received(:new) }
|
86
|
+
it { expect(worker.instance_variable_get(:@promises).size).to eq(1) }
|
87
|
+
end
|
88
|
+
|
89
|
+
context 'with batch exhausted request' do
|
90
|
+
let(:batch_can_retry) { false }
|
91
|
+
let(:response) { nil }
|
92
|
+
|
93
|
+
it { expect(LiveQA::Request).to have_received(:execute) }
|
94
|
+
it { expect(batch).to have_received(:update_retry) }
|
95
|
+
it { expect(Concurrent::ScheduledTask).to_not have_received(:new) }
|
96
|
+
it { expect(worker.instance_variable_get(:@promises).size).to eq(0) }
|
97
|
+
end
|
24
98
|
end
|
99
|
+
|
25
100
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: liveqa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.9.
|
4
|
+
version: 1.9.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- LiveQA
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-08-
|
11
|
+
date: 2018-08-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|