receptor_controller-client 0.0.2 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1545985069380f37a6e78393b779ec86c3d189065c2a7086098128f5ff1f7f42
-  data.tar.gz: 0a8a95476224fa7ed5f0f7cbca7b1b034b6ad2b0f8ae89a07ff987e80ffbdd4a
+  metadata.gz: ab72f29ace5c188996000048957bb58020940d3b8ce32b1a0b720602f4ff7e64
+  data.tar.gz: 4b68d648d59809f80e108a1d998af10d09fee233db085e02b736c691db05b8d5
 SHA512:
-  metadata.gz: d78e3987b4436a930fa9bc8e5c6a727480202060ce3d9bd48b7e1b7bbe3b60874ea57191e0cd4610257a5c0b4ae1c747e01b662ca356b7050176f4727c06680f
-  data.tar.gz: c7a7934d2aacf9fd83ce41fb38d5e536f380378e51857a3b1dc0a6ca0192a0073ff07dd7dccb8091fff65850ff2cc6d2f60aafa4308d4e78b7f69efa9e30fad4
+  metadata.gz: 32b8c1c4a2173e38ec229bad3b4f0c6747c9f9cc50ce3be86746217a6682b60e4c820e9677adc38798548d81bb62db858994a7019cb85fc9b58c149eb0357c2e
+  data.tar.gz: abfdf8a4c9924f4ba225310f5439b97f003d4ac01eedd14a822328d7750289a60cc9ec49a086ac125f446adbd0c84840afc02562a84c3c02c19f6402b201a469

data/CHANGELOG.md

@@ -5,6 +5,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.0.7]
+- rdkafka changes #32
+
+## [0.0.6]
+- Waiting for response-worker kafka init #20
+- Fix href_slug in directives log #29
+
+## [0.0.5] - 2020-08-11
+### Added
+- Unique Persist-ref for every response worker #18
+
+## [0.0.4] - 2020-08-04
+### Added
+- Default Kafka ACK to true
+
+## [0.0.3] - 2020-08-03
+### Added
+- Do not send EOF message until all messages have been received
 
 ## [0.0.2] - 2020-07-31
 ### Initial release to rubygems.org
@@ -12,5 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Added extra logging for messages
 
-[Unreleased]: https://github.com/RedHatInsights/receptor_controller-client-ruby/compare/v0.0.2...HEAD
+[Unreleased]: https://github.com/RedHatInsights/receptor_controller-client-ruby/compare/v0.0.7...HEAD
+[0.0.6]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.7
+[0.0.6]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.6
+[0.0.5]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.5
+[0.0.4]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.4
+[0.0.3]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.3
 [0.0.2]: https://github.com/RedHatInsights/receptor_controller-client-ruby/releases/tag/v0.0.2

data/lib/receptor_controller/client/configuration.rb

@@ -39,10 +39,10 @@ module ReceptorController
       @job_path               = '/job'
       @pre_shared_key         = nil
 
-      @queue_auto_ack    = false
+      @queue_auto_ack    = true
       @queue_host        = nil
       @queue_max_bytes   = nil
-      @queue_persist_ref = nil
+      @queue_persist_ref = "receptor-client.#{SecureRandom.uuid}"
       @queue_port        = nil
       @queue_topic       = 'platform.receptor-controller.responses'
 

data/lib/receptor_controller/client/directive_blocking.rb

@@ -20,7 +20,7 @@ module ReceptorController
 
       msg_id = JSON.parse(response.body)['id']
 
-      logger.debug("Receptor response: registering message #{msg_id}".tap { |msg| msg << " req: #{body["href_slug"]}" if ENV["LOG_ALL_RECEPTOR_MESSAGES"]&.to_i != 0 })
+      logger.debug("Receptor response [#{ReceptorController::Client::Configuration.default.queue_persist_ref}]: registering message #{msg_id}, href_slug: #{log_message_common}")
       # registers message id for kafka responses
       response_worker.register_message(msg_id, self)
       wait_for_response(msg_id)

data/lib/receptor_controller/client/directive_non_blocking.rb

@@ -48,6 +48,7 @@ module ReceptorController
 
         # registers message id for kafka responses
         response_worker.register_message(msg_id, self)
+        logger.debug("Receptor response [#{ReceptorController::Client::Configuration.default.queue_persist_ref}]: registering message #{msg_id}, href_slug: #{log_message_common}")
 
         msg_id
       else

data/lib/receptor_controller/client/response_worker.rb

@@ -1,3 +1,4 @@
+require 'active_support/notifications'
 require 'base64'
 require "concurrent"
 require 'stringio'
@@ -18,6 +19,10 @@ module ReceptorController
   #
   # Use "start" and "stop" methods to start/stop listening on Kafka
   class Client::ResponseWorker
+    EOF = "eof".freeze
+    RESPONSE = "response".freeze
+    INITIALIZATION_TIMEOUT = 20
+
     attr_reader :started
     alias started? started
 
@@ -36,13 +41,19 @@ module ReceptorController
 
     # Start listening on Kafka
     def start
+      init_lock, init_wait = Mutex.new, ConditionVariable.new
+
       lock.synchronize do
         return if started.value
 
+        default_messaging_opts # Thread-safe init
+
         started.value         = true
         workers[:maintenance] = Thread.new { check_timeouts while started.value }
         workers[:listener]    = Thread.new { listen while started.value }
       end
+
+      logger.info("Receptor Response worker started...")
     end
 
     # Stop listener
@@ -81,12 +92,11 @@ module ReceptorController
       # Open a connection to the messaging service
       client = ManageIQ::Messaging::Client.open(default_messaging_opts)
 
-      logger.info("Receptor Response worker started...")
       client.subscribe_topic(queue_opts) do |message|
         process_message(message)
       end
     rescue => err
-      logger.error("Exception in kafka listener: #{err}\n#{err.backtrace.join("\n")}")
+      logger.error(response_log("Exception in kafka listener: #{err}\n#{err.backtrace.join("\n")}"))
     ensure
       client&.close
     end
@@ -95,7 +105,7 @@ module ReceptorController
       response = JSON.parse(message.payload)
 
       if (message_id = response['in_response_to'])
-        logger.debug("Receptor response: Received message_id: #{message_id}")
+        logger.debug(response_log("Received message #{message_id}: serial: #{response["serial"]}, type: #{response['message_type']}, payload: #{response['payload'] || "n/a"}"))
         if (callbacks = registered_messages[message_id]).present?
           # Reset last_checked_at to avoid timeout in multi-response messages
           reset_last_checked_at(callbacks)
@@ -105,35 +115,52 @@ module ReceptorController
             # Response OK
             #
             message_type = response['message_type'] # "response" (with data) or "eof" (without data)
-            registered_messages.delete(message_id) if message_type == 'eof'
-
             payload = response['payload']
-            payload = unpack_payload(payload) if message_type == 'response' && payload.kind_of?(String)
-
-            logger.debug("Receptor response: OK | message #{message_id} (#{payload})")
-
-            callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
+            callbacks[:received_msgs] ? callbacks[:received_msgs] += 1 : callbacks[:received_msgs] = 1
+
+            case message_type
+            when EOF
+              # Store how many messages are needed to be received for this request
+              callbacks[:total_msgs] = response["serial"]
+            when RESPONSE
+              payload = unpack_payload(payload) if payload.kind_of?(String)
+              callbacks[:msg_size] ? callbacks[:msg_size] += payload.size : callbacks[:msg_size] = payload.size
+              callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
+            else
+              # Send the callback to release the thread.
+              logger.warn(response_log("Unexpected type | message #{message_id}, type: #{message_type}"))
+              callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
+            end
+
+            # We received all the messages, complete the message.
+            if callbacks[:received_msgs] == callbacks[:total_msgs]
+              registered_messages.delete(message_id)
+              callbacks[:receiver].send(callbacks[:response_callback], message_id, EOF, payload)
+              logger.debug(response_log("Message #{message_id} complete, total bytes: #{callbacks[:msg_size]}"))
+            end
+
+            logger.debug(response_log("OK | message: #{message_id}, serial: #{response["serial"]}, type: #{message_type}, payload: #{payload || "n/a"}"))
           else
             #
             # Response Error
             #
             registered_messages.delete(message_id)
 
-            logger.error("Receptor response: ERROR | message #{message_id} (#{response})")
+            logger.error(response_log("ERROR | message #{message_id} (#{response})"))
 
             callbacks[:receiver].send(callbacks[:error_callback], message_id, response['code'], response['payload'])
           end
         elsif ENV["LOG_ALL_RECEPTOR_MESSAGES"]&.to_i != 0
           # noop, it's not error if not registered, can be processed by another pod
-          logger.debug("Receptor response unhandled: #{message_id} (#{response['code']})")
+          logger.debug(response_log("NOT REGISTERED | #{message_id} (#{response['code']})"))
         end
       else
-        logger.error("Receptor response: Message id (in_response_to) not received! #{response}")
+        logger.error(response_log("MISSING | Message id (in_response_to) not received! #{response}"))
       end
     rescue JSON::ParserError => e
-      logger.error("Receptor response: Failed to parse Kafka response (#{e.message})\n#{message.payload}")
+      logger.error(response_log("Failed to parse Kafka response (#{e.message})\n#{message.payload}"))
     rescue => e
-      logger.error("Receptor response: #{e}\n#{e.backtrace.join("\n")}")
+      logger.error(response_log("#{e}\n#{e.backtrace.join("\n")}"))
     ensure
       message.ack unless config.queue_auto_ack
     end
@@ -194,21 +221,30 @@ module ReceptorController
     end
 
     # No persist_ref here, because all instances (pods) needs to receive kafka message
+    # TODO: temporary changed to unique persist_ref
     def queue_opts
-      opts               = {:service  => config.queue_topic,
-                            :auto_ack => config.queue_auto_ack}
-      opts[:max_bytes]   = config.queue_max_bytes if config.queue_max_bytes
-      opts[:persist_ref] = config.queue_persist_ref if config.queue_persist_ref
-      opts
+      return @queue_opts if @queue_opts
+
+      @queue_opts               = {:service  => config.queue_topic,
+                                   :auto_ack => config.queue_auto_ack}
+      @queue_opts[:max_bytes]   = config.queue_max_bytes if config.queue_max_bytes
+      @queue_opts[:persist_ref] = ENV['HOSTNAME']
+      @queue_opts
     end
 
     def default_messaging_opts
-      {
+      return @default_messaging_opts if @default_messaging_opts
+
+      @default_messaging_opts = {
         :host       => config.queue_host,
         :port       => config.queue_port,
         :protocol   => :Kafka,
         :client_ref => "receptor_client-responses-#{Time.now.to_i}", # A reference string to identify the client
       }
     end
+
+    def response_log(message)
+      "Receptor Response [#{queue_opts[:persist_ref]}]: #{message}"
+    end
   end
 end

data/lib/receptor_controller/client/version.rb

@@ -1,5 +1,5 @@
 module ReceptorController
   class Client
-    VERSION = "0.0.2".freeze
+    VERSION = "0.0.7".freeze
   end
 end

data/receptor_controller-client.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'faraday', '~> 1.0'
   s.add_runtime_dependency 'json', '~> 2.3', '>= 2.3.0'
   s.add_runtime_dependency 'manageiq-loggers', '~> 0.5.0'
-  s.add_runtime_dependency 'manageiq-messaging', '~> 0.1.5'
+  s.add_runtime_dependency 'manageiq-messaging', '~> 1.0.0'
 
   s.add_development_dependency 'bundler', '~> 2.0'
   s.add_development_dependency 'rake', '>= 12.3.3'

data/spec/receptor_controller/directive_blocking_spec.rb

@@ -32,6 +32,10 @@ RSpec.describe ReceptorController::Client::DirectiveBlocking do
 
   subject { described_class.new(:name => directive, :account => external_tenant, :node_id => receptor_node, :payload => payload, :client => receptor_client) }
 
+  before do
+    allow(subject.response_worker).to receive_messages(:init_notifications => nil, :wait_for_notifications => nil)
+  end
+
   describe "#call" do
     it "makes POST /job request to receptor and registers received message ID" do
       allow(subject).to receive(:wait_for_response)
@@ -106,7 +110,8 @@ RSpec.describe ReceptorController::Client::DirectiveBlocking do
         response_message = {'code'           => 0,
                             'in_response_to' => http_response['id'],
                             'message_type'   => subject.class::MESSAGE_TYPE_EOF,
-                            'payload'        => 'Unimportant'}
+                            'payload'        => 'Unimportant',
+                            'serial'         => 1}
 
         allow(kafka_response).to receive(:payload).and_return(response_message.to_json)
 

data/spec/receptor_controller/directive_non_blocking_spec.rb

@@ -101,13 +101,13 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
       # Kafka response - 'response' type
       expect(subject).to receive(:response_success).twice.and_call_original
       message_type, payload = 'response', 'Testing payload'
-      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload}
+      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload, :serial => 1}
       allow(response).to receive(:payload).and_return(response_payload.to_json)
       subject.response_worker.send(:process_message, response)
 
       # Kafka response - 'eof' type
       message_type = 'eof'
-      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil}
+      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil, :serial => 2}
       allow(response).to receive(:payload).and_return(response_payload.to_json)
       subject.response_worker.send(:process_message, response)
 
@@ -142,7 +142,7 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
 
       # Kafka response - 'response' type - success
       message_type, payload = 'response', 'Testing payload'
-      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload}
+      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => payload, :serial => 1}
       allow(response).to receive(:payload).and_return(response_payload.to_json)
       success_calls.times do
         subject.response_worker.send(:process_message, response)
@@ -150,7 +150,7 @@ RSpec.describe ReceptorController::Client::DirectiveNonBlocking do
 
       # Kafka response - 'eof' type
       message_type = 'eof'
-      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil}
+      response_payload = {'code' => 0, 'in_response_to' => response_id, 'message_type' => message_type, 'payload' => nil, :serial => 4}
       allow(response).to receive(:payload).and_return(response_payload.to_json)
       subject.response_worker.send(:process_message, response)
 

data/spec/receptor_controller/response_worker_spec.rb

@@ -9,7 +9,13 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
 
   before do
     allow(logger).to receive_messages(%i[debug info warn error fatal])
-    allow(config).to receive_messages(:queue_auto_ack => false, :response_timeout => 0, :response_timeout_poll_time => 0)
+    allow(config).to receive_messages(:queue_auto_ack => false,
+                                      :queue_max_bytes => nil,
+                                      :queue_persist_ref => 'consumer-group-1',
+                                      :queue_topic => 'receptor.kafka.topic',
+                                      :response_timeout => 0,
+                                      :response_timeout_poll_time => 0,
+                                      )
   end
 
   describe "#register_message" do
@@ -87,7 +93,7 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
         end
 
         it "logs error" do
-          expect(logger).to receive(:error).with(/Receptor response: Failed to parse Kafka response/)
+          expect(logger).to receive(:error).with(/Failed to parse Kafka response/)
 
           subject.send(:process_message, message)
         end
@@ -97,7 +103,7 @@ RSpec.describe ReceptorController::Client::ResponseWorker do
         let(:payload) { {'code' => 0, 'message_type' => 'response', 'payload' => response_body} }
 
         it "logs error" do
-          expect(logger).to receive(:error).with(/Receptor response: Message id \(in_response_to\) not received!/)
+          expect(logger).to receive(:error).with(/Message id \(in_response_to\) not received!/)
 
           subject.send(:process_message, message)
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: receptor_controller-client
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.7
 platform: ruby
 authors:
 - Martin Slemr
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-31 00:00:00.000000000 Z
+date: 2020-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -98,14 +98,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -261,8 +261,8 @@ signing_key:
 specification_version: 4
 summary: Client for communication with Platform Receptor Controller - Gem
 test_files:
-- spec/receptor_controller/directive_blocking_spec.rb
+- spec/receptor_controller/directive_non_blocking_spec.rb
 - spec/receptor_controller/response_worker_spec.rb
+- spec/receptor_controller/directive_blocking_spec.rb
 - spec/receptor_controller/client_spec.rb
-- spec/receptor_controller/directive_non_blocking_spec.rb
 - spec/spec_helper.rb