liveqa 1.9.3 → 1.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00f9398c7481931cbc0b9855ae4d5020b8d66734
-  data.tar.gz: 3c736aea5751dbaf497deca35748026724c23fd3
+  metadata.gz: 46d9bc855dce533eaf940cc977b28d8904fed773
+  data.tar.gz: eaf6c87c0118f7a48e3e699101653fb99c89cf9a
 SHA512:
-  metadata.gz: ebf4ba052d2515b16ded6e24c5df4bcb137afbacd6076f376246f76a89acf1e13a5a42494d9f7b48f95e040b854f70072bfc3f6137847495c4ab7a826e8ba23e
-  data.tar.gz: 3c608f34f6802396cb6ae478098d4e7018c8d1c62e520fea8f524861bc84a0ddfa0c923abd4b5e059e18e271b12abcac6a86a98b826b33b3ad319ec59e9f5bbb
+  metadata.gz: cb08f5e030859cd1c0b1939b0c3d325a1e35b77e916c2358d08135221c1265a047f7d396e322dd8662002b3e47d1c25d379e9c35631e21faed18b0edc4f6d26f
+  data.tar.gz: 5dce2d321778216a67beab91f2243ba167bbcb12612d0d3aec024d0952e81527320e50eeb50a7e95d06b666f45329f843c0bd59111709c1bc5e43e3cd764ccbc

data/LICENCE

@@ -1,6 +1,6 @@
 The MIT License
 
-Copyright (c) 2018- LiveQA, Inc. (https://www.liveqa.com)
+Copyright (c) 2018- LiveQA, Inc. (https://www.liveqa.io)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

data/lib/liveqa/processor/async.rb

@@ -3,29 +3,34 @@ module LiveQA
     class Async
 
       def initialize
-        @max_queue_size = 10_000
-        @queue = Concurrent::Array.new
-        @worker = Worker.new(@queue, executor)
+        @queue = Queue.new
+        @state_worker = Concurrent::AtomicBoolean.new(true)
+        @worker = Worker.new(@queue, @state_worker)
 
         at_exit do
-          if worker_running?
-            executor.shutdown
-            executor.wait_for_termination
-          end
+          shutdown_worker if worker_running?
         end
       end
 
       def enqueue(attributes)
-        return false if @queue.length > @max_queue_size
+        ensure_worker_running
 
         @queue << attributes
-        ensure_worker_running
 
         true
       end
 
       private
 
+      def shutdown_worker
+        @state_worker.make_false
+        @queue << LiveQA::Processor::Worker::SHUTDOWN_MESSAGE
+        @worker_thread.wait
+
+        executor.shutdown
+        executor.wait_for_termination
+      end
+
       def executor
         @executor ||= Concurrent.global_io_executor
       end

data/lib/liveqa/processor/batch.rb

@@ -8,6 +8,8 @@ module LiveQA
       MAX_MESSAGES = 100
       MAX_MESSAGE_BYTES = 32_768 # 32Kb
 
+      DEFAULT_RETRY_TIME = 60
+
       MAX_RETRY = 10
       RETRY_MAP = {
         1 => 2,
@@ -43,19 +45,20 @@ module LiveQA
         true
       end
 
+      def empty?
+        @messages.size.zero?
+      end
+
       def full?
         max_messages_reached? || max_size_reached?
       end
 
-      def update_retry
-        @retry_count += 1
-        @retry_at = Time.now + (RETRY_MAP[@retry_count] || 120)
+      def next_retry
+        RETRY_MAP[@retry_count] || DEFAULT_RETRY_TIME
       end
 
-      def can_run?
-        return true if @retry_count.zero?
-
-        Time.now >= @retry_at
+      def update_retry
+        @retry_count += 1
       end
 
       def can_retry?

data/lib/liveqa/processor/worker.rb

@@ -2,67 +2,110 @@ module LiveQA
   module Processor
     class Worker
 
-      DEFAULT_WAIT_TIME = 2
+      FLUSH_INTERVAL_SECONDS = 3
 
-      attr_reader :queue
-      attr_reader :batches
-      attr_reader :executor
+      FLUSH_MESSAGE = Object.new
+      SHUTDOWN_MESSAGE = Object.new
 
-      def initialize(queue, executor)
+      def initialize(queue, state)
         @queue = queue
-        @batches = []
+        @state = state
 
-        @executor = executor
+        @batch = Batch.new
+        @promises = Concurrent::Array.new
+
+        @timer = Concurrent::TimerTask.new(execution_interval: FLUSH_INTERVAL_SECONDS) { @queue << FLUSH_MESSAGE }
+        @timer.execute
       end
 
       def run
-        while running?
-          return if queue.empty? && batches.empty?
-
-          sleep(DEFAULT_WAIT_TIME) if queue.empty?
+        while thread_active?
+          message = @queue.pop
 
-          create_new_batch unless queue.empty?
-
-          send_batches
+          add_message_to_batch(message)
         end
+
+        shutdown_worker
       end
 
       private
 
-      def running?
-        return true if !queue.empty? || !batches.empty?
-        executor.running?
+      ##
+      # Add the message to a batch
+      #
+      # - Add message to a batch if it's not a flush message
+      # - flush batch if flush message is receive or batch is full
+      def add_message_to_batch(message)
+        @batch << message if message != FLUSH_MESSAGE
+        flush if message == FLUSH_MESSAGE || @batch.full?
       end
 
-      def create_new_batch
-        batch = Batch.new
-        batch << queue.pop until batch.full? || queue.empty?
-        batches.push(batch)
+      ##
+      # Shutdown the worker
+      #
+      # - Close the timer
+      # - Retreive the last messages
+      # - Wait all the request to be completed
+      #
+      # rubocop:disable Lint/HandleExceptions
+      def shutdown_worker
+        @timer.shutdown
+
+        begin
+          until (message = @queue.pop(true)).nil?
+            add_message_to_batch(message)
+          end
+        rescue ThreadError => _
+        end
+
+        flush
+
+        @promises.each do |promise|
+          promise.wait if promise && !promise.fulfilled?
+        end
       end
+      # rubocop:enable Lint/HandleExceptions
 
-      def send_batches
-        batches.dup.each do
-          batch = batches.pop
+      def flush
+        return if @batch.empty?
 
-          unless batch.can_run?
-            batches.push(batch)
-            next
-          end
+        send_batch(@batch)
+        @batch = Batch.new
+      end
 
-          begin
-            LiveQA::Batch.create(Message.base.merge(data: batch.messages))
-          rescue LiveQA::RequestError => error
-            return LiveQA.configurations.logger.error(error.message) if [401, 404].include?(error.http_status)
-            batch_retry(batch)
-          rescue Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => _error
-            batch_retry(batch)
+      # rubocop:disable Metrics/AbcSize
+      def send_batch(batch)
+        promise =
+          Concurrent::Promise
+          .new { LiveQA::Batch.create(Message.base.merge(data: batch.messages)) }
+          .on_success { @promises.delete(promise) }
+          .rescue do |error|
+            batch.update_retry
+
+            if error.is_a?(LiveQA::RequestError) && [401, 404].include?(error.http_status)
+              return logger.error(error.message)
+            end
+
+            if thread_active? && batch.can_retry?
+              Concurrent::ScheduledTask.new(batch.next_retry) { send_batch(batch) }.execute
+            else
+              @promises.delete(promise)
+            end
           end
-        end
+
+        promise.execute
+        @promises << promise
+      rescue Concurrent::RejectedExecutionError => _
+        logger.error('Impossible to start a thread in closing application')
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def thread_active?
+        @state.true?
       end
 
-      def batch_retry(batch)
-        batch.update_retry
-        batches.push(batch) if batch.can_retry?
+      def logger
+        LiveQA.configurations.logger
       end
 
     end

data/lib/liveqa/version.rb

@@ -1,3 +1,3 @@
 module LiveQA # :nodoc:
-  VERSION = '1.9.3'.freeze
+  VERSION = '1.9.4'.freeze
 end

data/spec/lib/liveqa/batch_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe LiveQA::Batch do
 
   describe '.create' do
-    let(:response) { double('LiveQA::Request', body:  "{\"object\":\"event\",\"id\":41}") }
+    let(:response) { double('LiveQA::Request', body:  "{\"object\":\"batch\",\"id\":41}") }
     before { allow(LiveQA::Request).to receive(:execute).and_return(response) }
 
     subject(:create) { LiveQA::Batch.create(data: []) }

data/spec/lib/liveqa/processor/async_spec.rb

@@ -5,7 +5,6 @@ describe LiveQA::Processor::Async do
   let(:processor) { LiveQA::Processor::Async.new }
 
   context 'enqueue a message' do
-
     before do
       allow(processor).to receive(:ensure_worker_running)
       processor.enqueue({ test: 'hello' })
@@ -15,5 +14,23 @@ describe LiveQA::Processor::Async do
     it { expect(processor.instance_variable_get(:@queue).length).to eq(1) }
   end
 
+  context 'shutting down worker' do
+    before do
+      allow_message_expectations_on_nil
+
+      allow(processor.instance_variable_get(:@state_worker)).to receive(:make_false)
+      allow(processor.instance_variable_get(:@worker_thread)).to receive(:wait)
+      allow(Concurrent.global_io_executor).to receive(:shutdown)
+      allow(Concurrent.global_io_executor).to receive(:wait_for_termination)
+
+      processor.send(:shutdown_worker)
+    end
+
+    it { expect(processor.instance_variable_get(:@state_worker)).to have_received(:make_false) }
+    it { expect(processor.instance_variable_get(:@worker_thread)).to have_received(:wait) }
+    it { expect(Concurrent.global_io_executor).to have_received(:shutdown) }
+    it { expect(Concurrent.global_io_executor).to have_received(:wait_for_termination) }
+    it { expect(processor.instance_variable_get(:@queue).length).to eq(1) }
+  end
 
 end

data/spec/lib/liveqa/processor/batch_spec.rb

@@ -5,9 +5,9 @@ describe LiveQA::Processor::Batch do
 
   context 'set defaults' do
     it { expect(batch.messages).to eq([]) }
-    it { expect(batch.can_run?).to be_truthy }
     it { expect(batch.can_retry?).to be_truthy }
     it { expect(batch.full?).to be_falsey }
+    it { expect(batch.empty?).to be_truthy }
   end
 
   context 'add message' do
@@ -15,13 +15,14 @@ describe LiveQA::Processor::Batch do
 
     it { expect(batch.messages).to eq([{ test: 'hello' }]) }
     it { expect(batch.full?).to be_falsey }
+    it { expect(batch.empty?).to be_falsey }
   end
 
   context 'with extended task' do
     before { batch.update_retry }
 
-    it { expect(batch.can_run?).to be_falsey }
     it { expect(batch.can_retry?).to be_truthy }
+    it { expect(batch.next_retry).to eq(2) }
   end
 
   context 'with extended task' do
@@ -29,20 +30,22 @@ describe LiveQA::Processor::Batch do
       11.times { batch.update_retry }
     end
 
-    it { expect(batch.can_run?).to be_falsey }
     it { expect(batch.can_retry?).to be_falsey }
+    it { expect(batch.next_retry).to eq(60) }
   end
 
   context 'with max number of messages reached' do
     before { 100.times { batch << { test: 'hello' }}}
 
     it { expect(batch.full?).to be_truthy }
+    it { expect(batch.empty?).to be_falsey }
   end
 
   context 'with max size reached' do
     before { 20.times { batch << 1000.times.map {{ test: 'hello' }}}}
 
     it { expect(batch.full?).to be_truthy }
+    it { expect(batch.empty?).to be_falsey }
   end
 
 end

data/spec/lib/liveqa/processor/worker_spec.rb

@@ -1,25 +1,100 @@
 require 'spec_helper'
 
 describe LiveQA::Processor::Worker do
+  let(:state_worker) { Concurrent::AtomicBoolean.new(true) }
   let(:queue) { Queue.new }
-  let(:worker) { LiveQA::Processor::Worker.new(queue, Concurrent.global_io_executor) }
+  let(:worker) { LiveQA::Processor::Worker.new(queue, state_worker) }
+  let!(:current_batch) { worker.instance_variable_get(:@batch) }
 
-  context 'do nothing' do
+  describe 'private#add_message_to_batch' do
+    context 'add to batch' do
+      let(:message) { Object.new }
+      before { worker.send(:add_message_to_batch, message) }
+
+      it { expect(current_batch.messages).to eq([message]) }
+    end
+
+    context 'with batch full' do
+      before do
+        allow(current_batch).to receive(:full?).and_return(true)
+        allow(worker).to receive(:flush)
+        worker.send(:add_message_to_batch, Object.new)
+      end
+
+      it { expect(worker).to have_received(:flush) }
+    end
+
+    context 'call flush' do
+      before do
+        allow(worker).to receive(:flush)
+        worker.send(:add_message_to_batch, LiveQA::Processor::Worker::FLUSH_MESSAGE)
+      end
+
+      it { expect(worker).to have_received(:flush) }
+    end
+  end
+
+  describe 'private#flush' do
     before do
-      allow(worker).to receive(:send_batches)
-      worker.run
+      allow(current_batch).to receive(:empty?).and_return(batch_status)
+      allow(worker).to receive(:send_batch)
+      worker.send(:flush)
+    end
+
+    context 'with not empty batch' do
+      let(:batch_status) { false }
+
+      it { expect(worker).to have_received(:send_batch).with(current_batch) }
+      it { expect(current_batch).to_not eq(worker.instance_variable_get(:@batch)) }
     end
 
-    it { expect(worker).not_to have_received(:send_batches) }
+    context 'with empty batch' do
+      let(:batch_status) { true }
+
+      it { expect(worker).to_not have_received(:send_batch) }
+      it { expect(current_batch).to eq(worker.instance_variable_get(:@batch)) }
+    end
   end
 
-  context 'with something in the queue' do
+  describe 'private#send_batch' do
+    let(:batch) { LiveQA::Processor::Batch.new }
+    let(:batch_can_retry) { true }
+
     before do
-      queue << { test: true }
-      allow(LiveQA::Batch).to receive(:create)
-      worker.run
+      allow(LiveQA::Request).to receive(:execute).and_return(response)
+      allow(batch).to receive(:update_retry)
+      allow(batch).to receive(:can_retry?).and_return(batch_can_retry)
+      allow(Concurrent::ScheduledTask).to receive(:new)
+      worker.send(:send_batch, batch)
+      sleep(0.1)
+    end
+
+    context 'with valid request' do
+      let(:response) { double('LiveQA::Request', body: "{\"object\":\"batch\",\"id\":41}") }
+
+      it { expect(LiveQA::Request).to have_received(:execute) }
+      it { expect(batch).to_not have_received(:update_retry) }
+      it { expect(worker.instance_variable_get(:@promises).size).to eq(0) }
     end
 
-    it { expect(LiveQA::Batch).to have_received(:create) }
+    context 'with invalid request' do
+      let(:response) { nil }
+
+      it { expect(LiveQA::Request).to have_received(:execute) }
+      it { expect(batch).to have_received(:update_retry) }
+      it { expect(Concurrent::ScheduledTask).to have_received(:new) }
+      it { expect(worker.instance_variable_get(:@promises).size).to eq(1) }
+    end
+
+    context 'with batch exhausted request' do
+      let(:batch_can_retry) { false }
+      let(:response) { nil }
+
+      it { expect(LiveQA::Request).to have_received(:execute) }
+      it { expect(batch).to have_received(:update_retry) }
+      it { expect(Concurrent::ScheduledTask).to_not have_received(:new) }
+      it { expect(worker.instance_variable_get(:@promises).size).to eq(0) }
+    end
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: liveqa
 version: !ruby/object:Gem::Version
-  version: 1.9.3
+  version: 1.9.4
 platform: ruby
 authors:
 - LiveQA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-01 00:00:00.000000000 Z
+date: 2018-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby