nulogy_message_bus_consumer 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2faa5c725b717d217b37a0c9019ecd3f3ae9d40885a2e3b1ce898e626bbb2d3c
-  data.tar.gz: 865137198c57cfd60bdea31c482a66240a284c34d4f1149033160c28fb2a6cef
+  metadata.gz: b0b77dbd208bb83d34c4881d00765baac92c4fd8ba36d5f0395e5adf479fd6ca
+  data.tar.gz: 74360efce2d94e148f1792aa7f4b6424c656553c6ca4452cf66583a84352a315
 SHA512:
-  metadata.gz: 82c5cf164680d27183b17b341b362c5f1ec70a26fecf79d4f8d982ea3506e107ac8e216604c5182469c627892f3c423336053a2502c8e10c574c85af5dd82402
-  data.tar.gz: d2109e0aefef9eecebfb57425316cf36dd7cdb92a4356253efb73271f2fed5e4c1ed59e56a46b44a2dc869dd3acb5285b88ae11d6d72b15f50a7506ed85058a2
+  metadata.gz: 4503a0fda0106a555a2063319ff1d71127503ab0638a50b4d2fa04e55b38d109039fb7863f1162a397140c15f255f36309d15bc2f6ce3d9f8d168a2f9479f42b
+  data.tar.gz: af0f4d1a905cdddfb8b04f7a4b3c232e66eb32d9b90daf28ede558afd4879e529c3992147bac19f75f4b097966a020ec87ff5013e40370ddcc43325fa50d1dd3

data/Rakefile

@@ -18,13 +18,15 @@ APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
 load "rails/tasks/engine.rake"
 load "rails/tasks/statistics.rake"
 
+require "rspec/core"
+require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 require "rubocop/rake_task"
 RuboCop::RakeTask.new
-task default: [:spec, :rubocop]
+task default: %i[spec rubocop]
 
 require "rake/release"
 
 Rake::Release::Task.load_all do |spec|
-  spec.version_tag = "nulogy_message_bus_producer-v#{spec.version}"
+  spec.version_tag = "nulogy_message_bus_consumer-v#{spec.version}"
 end

data/lib/nulogy_message_bus_consumer.rb

@@ -31,7 +31,7 @@ module NulogyMessageBusConsumer
   end
 
   def logger
-    @@logger ||= NullLogger.new
+    @logger ||= NullLogger.new
   end
 
   def invoke_pipeline(*steps)
@@ -56,7 +56,7 @@ module NulogyMessageBusConsumer
       Steps::ConnectToMessageBus.new(config, logger),
       Steps::SeekBeginningOfTopic.new,
       Steps::StreamMessagesUntilNoneAreLeft.new(logger),
-      Handlers::LogUnprocessedMessages.new(logger)
+      Handlers::LogUnprocessedMessages.new(logger),
     ])
   end
 end

data/lib/nulogy_message_bus_consumer/handlers/log_unprocessed_messages.rb

@@ -7,9 +7,10 @@ module NulogyMessageBusConsumer
 
       def call(message:, **_)
         return if ProcessedMessage.exists?(id: message.id)
+
         @logger.warn(JSON.dump(
           event: "unprocessed_message",
-          kafka_message: message.to_h,
+          kafka_message: message.to_h
         ))
       end
     end

data/lib/nulogy_message_bus_consumer/kafka_utils.rb

@@ -13,13 +13,14 @@ module NulogyMessageBusConsumer
     def wait_for(attempts: 100, interval: 0.1)
       attempts.times do
         break if yield
+
         sleep interval
       end
     end
 
     def every_message_until_none_are_left(consumer)
       Enumerator.new do |yielder|
-        while message = consumer.poll(250)
+        while (message = consumer.poll(250))
           yielder.yield(message)
         end
       end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -1,9 +1,5 @@
 module NulogyMessageBusConsumer
   class Message
-    def initialize(attrs = {})
-      attrs.each { |key, value| instance_variable_set("@#{key}", value) }
-    end
-
     attr_reader :event_data
     attr_reader :event_data_unparsed
     attr_reader :id
@@ -14,10 +10,20 @@ module NulogyMessageBusConsumer
     attr_reader :company_uuid
     attr_reader :timestamp
     attr_reader :topic
+    attr_reader :created_at
+
+    def initialize(attrs = {})
+      attrs.each { |key, value| instance_variable_set("@#{key}", value) }
+    end
 
     def self.from_kafka(kafka_message)
       envelope_data = JSON.parse(kafka_message.payload, symbolize_names: true)
-      event_data = JSON.parse(envelope_data[:event_json], symbolize_names: true) rescue {}
+      event_data =
+        begin
+          JSON.parse(envelope_data[:event_json], symbolize_names: true)
+        rescue StandardError
+          {}
+        end
 
       new(
         event_data: event_data,
@@ -30,6 +36,7 @@ module NulogyMessageBusConsumer
         company_uuid: envelope_data[:company_uuid] || envelope_data[:tenant_id],
         timestamp: kafka_message.timestamp,
         topic: kafka_message.topic,
+        created_at: envelope_data[:created_at]
       )
     end
 

data/lib/nulogy_message_bus_consumer/pipeline.rb

@@ -9,7 +9,7 @@ module NulogyMessageBusConsumer
     end
 
     def insert(step, after:)
-      index = @steps.find_index { |step| step.is_a?(after) }
+      index = @steps.find_index { |s| s.is_a?(after) }
       @steps.insert(index + 1, step)
     end
 
@@ -25,7 +25,7 @@ module NulogyMessageBusConsumer
       @steps.reverse.reduce(last_step) do |composed_steps, previous_step|
         lambda do |**args|
           invoke_next = compose_with_merged_args(args, composed_steps)
-            previous_step.call(**args, &invoke_next)
+          previous_step.call(**args, &invoke_next)
         end
       end
     end
@@ -33,7 +33,10 @@ module NulogyMessageBusConsumer
     def compose_with_merged_args(existing_args, func)
       lambda do |**yielded_args|
         args_to_be_overridden = existing_args.keys & yielded_args.keys
-        raise "Cannot override existing argument(s): #{args_to_be_overridden.join(', ')}" if args_to_be_overridden.any?
+        if args_to_be_overridden.any?
+          raise "Cannot override existing argument(s): #{args_to_be_overridden.join(', ')}"
+        end
+
         func.call(**existing_args.merge(yielded_args))
       end
     end

data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb

@@ -1,7 +1,7 @@
 module NulogyMessageBusConsumer
   module Steps
     class CommitOnSuccess
-      def call(kafka_consumer:, message:, **_)
+      def call(kafka_consumer:, **_)
         result = yield
 
         raise_if_invalid(result)

data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb

@@ -1,14 +1,15 @@
 module NulogyMessageBusConsumer
   module Steps
     class ConnectToMessageBus
-      def initialize(config, logger)
+      def initialize(config, logger, kafka_consumer: nil)
         @config = config
         @logger = logger
+        @kafka_consumer = kafka_consumer
       end
 
       def call(**_)
         @logger.info("Connecting to the MessageBus")
-        consumer = Rdkafka::Config.new(consumer_config).consumer
+        consumer = build_consumer
         @logger.info("Using consumer group id: #{@config.consumer_group_id}")
 
         consumer.subscribe(@config.topic_name)
@@ -22,6 +23,10 @@ module NulogyMessageBusConsumer
 
       private
 
+      def build_consumer
+        @kafka_consumer || Rdkafka::Config.new(consumer_config).consumer
+      end
+
       def consumer_config
         {
           "bootstrap.servers": @config.bootstrap_servers,

data/lib/nulogy_message_bus_consumer/steps/log_messages.rb

@@ -1,8 +1,16 @@
 module NulogyMessageBusConsumer
   module Steps
+    class Clock
+      # milliseconds since epoch
+      def now
+        Time.zone.now.to_datetime.strftime("%Q").to_i
+      end
+    end
+
     class LogMessages
-      def initialize(logger)
+      def initialize(logger, clock: Clock.new)
         @logger = logger
+        @clock = clock
       end
 
       def call(message:, **_)
@@ -14,15 +22,25 @@ module NulogyMessageBusConsumer
 
         result = yield
 
+        millis = diff_millis(message.created_at, @clock.now)
         @logger.info(JSON.dump({
           event: "message_processed",
           kafka_message_id: message.id,
           message: "Processed #{message.id}",
           result: result,
+          time_to_processed: millis,
         }))
 
         result
       end
+
+      # Debezium appears to be giving us nanos since epoch
+      # https://github.com/debezium/debezium/blob/5a115e902cdc1dc399ec02758dd1039a33e99bc2/debezium-core/src/main/java/io/debezium/jdbc/JdbcValueConverters.java#L237
+      def diff_millis(oldest_nanos, newest_millis)
+        old_millis = oldest_nanos / 1000
+
+        newest_millis - old_millis
+      end
     end
   end
 end

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/lib/tasks/engine/message_bus_consumer.rake

@@ -1,6 +1,6 @@
 namespace :message_bus_consumer do
   desc "Verifies that the messages in the message bus have been processed"
-  task :audit => :environment do
+  task audit: :environment do
     config = build_audit_config
 
     NulogyMessageBusConsumer
@@ -13,4 +13,4 @@ namespace :message_bus_consumer do
     audit_config.consumer_group_id = "#{audit_config.consumer_group_id}_consumer_audit"
     NulogyMessageBusConsumer.config
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nulogy_message_bus_consumer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Nulogy
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2020-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -52,6 +52,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -95,33 +137,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.9.0
 - !ruby/object:Gem::Dependency
-  name: rspec-rails
+  name: rspec-json_expectations
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: 2.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
-  name: rspec-json_expectations
+  name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 4.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 4.0.1
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -137,34 +179,34 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.81.0
 - !ruby/object:Gem::Dependency
-  name: rubocop-rspec
+  name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.38.1
+        version: 2.5.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.38.1
+        version: 2.5.2
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.3
+        version: 1.38.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.3
-description:
+        version: 1.38.1
+description: 
 email:
 - tass@nulogy.com
 executables: []
@@ -197,7 +239,7 @@ homepage: https://github.com/nulogy/message-bus/tree/master/gems/nulogy_message_
 licenses: []
 metadata:
   allowed_push_host: https://rubygems.org/
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -213,7 +255,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Code for accessing the Nulogy Message Bus
 test_files: []