karafka 2.0.0.beta1 → 2.0.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f108cb4288d0ed0510381f51c77d49e052b947f6180c9b9c0b06e0ac2b599894
-  data.tar.gz: 3d79066d0107c08f450ca9f4c3b5c4a39aae497836c80bf8380c65f1406b82c0
+  metadata.gz: 7f75623e7d9cdcc4ba151ad551079275528c56bf66cd9c32ecc585756a8d505c
+  data.tar.gz: e0becf53133b579f581ddfdf947bbff21221fe69a8c73a0406174aecd0155f3a
 SHA512:
-  metadata.gz: 4aae257010c992c59ce4b01ead54ff2cfd4e8ccd8cbe6b52214b3cedf8f879690e0d577f2b41f44b1ab6888d7e27bbc92f3ba4a69e8b127687fb4c43bff51fbc
-  data.tar.gz: f65e425cb84152d20a055bdb9a94fd98280597cdf5e431337cb8604040534cacbfdd03efd6dc23b86c9ecf25721c860bd55ca75ad3f98e4c66136a88c1efc4e7
+  metadata.gz: 2ef2ac59f1ea60136abbaccf460206a0c2f6d4fe3124eda520f3a19568702acc774e0d9e02eae24cfe6bb6cb8ee8aa74602588aa818dc3537cd6bbc8409f159d
+  data.tar.gz: e29e964e777e2bd8a551458f591b92aea69a3ac2eafa1bc1d75bb42d7cd6bb904abf997724be4268ae1f9d429627d7b92d5c38e51817d36b0f27c6499a062af3

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Karafka framework changelog
 
+## 2.0.0-beta2 (2022-06-07)
+- Abstract away notion of topics groups (until now it was just an array)
+- Optimize how jobs queue is closed. Since we enqueue jobs only from the listeners, we can safely close jobs queue once listeners are done. By extracting this responsibility from listeners, we remove corner cases and race conditions. Note here: for non-blocking jobs we do wait for them to finish while running the `poll`. This ensures, that for async jobs that are long-living, we do not reach `max.poll.interval`.
+- `Shutdown` jobs are executed in workers to align all the jobs behaviours.
+- `Shutdown` jobs are always blocking.
+- Notion of `ListenersBatch` was introduced similar to `WorkersBatch` to abstract this concept.
+- Change default `shutdown_timeout` to be more than `max_wait_time` not to cause forced shutdown when no messages are being received from Kafka.
+- Abstract away scheduling of revocation and shutdown jobs for both default and pro schedulers
+- Introduce a second (internal) messages buffer to distinguish between raw messages buffer and karafka messages buffer
+- Move messages and their metadata remap process to the listener thread to allow for their inline usage
+- Change how we wait in the shutdown phase, so shutdown jobs can still use Kafka connection even if they run for a longer period of time. This will prevent us from being kicked out from the group early.
+- Introduce validation that ensures, that `shutdown_timeout` is more than `max_wait_time`. This will prevent users from ending up with a config that could lead to frequent forceful shutdowns.
+
 ## 2.0.0-beta1 (2022-05-22)
 - Update the jobs queue blocking engine and allow for non-blocking jobs execution
 - Provide `#prepared` hook that always runs before the fetching loop is unblocked

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.beta1)
+    karafka (2.0.0.beta2)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)

data/config/errors.yml

@@ -2,6 +2,7 @@ en:
   dry_validation:
     errors:
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
+      shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
       topics_names_not_unique: all topic names within a single consumer group must be unique
       required_usage_count: Given topic must be used at least once
       consumer_groups_inclusion: Unknown consumer group

data/lib/active_job/karafka.rb

@@ -14,8 +14,8 @@ begin
   # We extend routing builder by adding a simple wrapper for easier jobs topics defining
   # This needs to be extended here as it is going to be used in karafka routes, hence doing that in
   # the railtie initializer would be too late
-  ::Karafka::Routing::Builder.include ::Karafka::ActiveJob::RoutingExtensions
-  ::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::RoutingExtensions
+  ::Karafka::Routing::Builder.include ::Karafka::ActiveJob::Routing::Extensions
+  ::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::Routing::Extensions
 rescue LoadError
   # We extend ActiveJob stuff in the railtie
 end

data/lib/karafka/active_job/routing/extensions.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  # ActiveJob related Karafka stuff
+  module ActiveJob
+    # Karafka routing ActiveJob related components
+    module Routing
+      # Routing extensions for ActiveJob
+      module Extensions
+        # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
+        # the consumer class
+        # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+        def active_job_topic(name)
+          topic(name) do
+            consumer App.config.internal.active_job.consumer
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/base_consumer.rb

@@ -27,7 +27,7 @@ module Karafka
         pause_tracker.reset
 
         # Mark as consumed only if manual offset management is not on
-        return if topic.manual_offset_management
+        next if topic.manual_offset_management
 
         # We use the non-blocking one here. If someone needs the blocking one, can implement it
         # with manual offset management

data/lib/karafka/connection/client.rb

@@ -30,7 +30,7 @@ module Karafka
         @mutex = Mutex.new
         @closed = false
         @subscription_group = subscription_group
-        @buffer = MessagesBuffer.new
+        @buffer = RawMessagesBuffer.new
         @rebalance_manager = RebalanceManager.new
         @kafka = build_consumer
         # Marks if we need to offset. If we did not store offsets, we should not commit the offset

data/lib/karafka/connection/listener.rb

@@ -3,9 +3,13 @@
 module Karafka
   module Connection
     # A single listener that listens to incoming messages from a single subscription group.
-    # It polls the messages and then enqueues. It also takes care of potential recovery from
+    # It polls the messages and then enqueues jobs. It also takes care of potential recovery from
     # critical errors by restarting everything in a safe manner.
+    #
+    # This is the heart of the consumption process.
     class Listener
+      include Helpers::Async
+
       # @param subscription_group [Karafka::Routing::SubscriptionGroup]
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
@@ -17,6 +21,10 @@ module Karafka
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
         # We reference scheduler here as it is much faster than fetching this each time
         @scheduler = ::Karafka::App.config.internal.scheduler
+        # We keep one buffer for messages to preserve memory and not allocate extra objects
+        # We can do this that way because we always first schedule jobs using messages before we
+        # fetch another batch.
+        @messages_buffer = MessagesBuffer.new(subscription_group)
       end
 
       # Runs the main listener fetch loop.
@@ -53,33 +61,55 @@ module Karafka
           )
 
           resume_paused_partitions
+
           # We need to fetch data before we revoke lost partitions details as during the polling
           # the callbacks for tracking lost partitions are triggered. Otherwise we would be always
           # one batch behind.
-          messages_buffer = @client.batch_poll
+          poll_and_remap_messages
 
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop.received',
             caller: self,
-            messages_buffer: messages_buffer
+            messages_buffer: @messages_buffer
           )
 
           # If there were revoked partitions, we need to wait on their jobs to finish before
           # distributing consuming jobs as upon revoking, we might get assigned to the same
           # partitions, thus getting their jobs. The revoking jobs need to finish before
           # appropriate consumers are taken down and re-created
-          wait(@subscription_group) if schedule_revoke_lost_partitions_jobs
-
-          schedule_partitions_jobs(messages_buffer)
+          build_and_schedule_revoke_lost_partitions_jobs
 
           # We wait only on jobs from our subscription group. Other groups are independent.
-          wait(@subscription_group)
+          wait
+
+          build_and_schedule_consumption_jobs
+
+          wait
 
           # We don't use the `#commit_offsets!` here for performance reasons. This can be achieved
           # if needed by using manual offset management.
           @client.commit_offsets
         end
 
+        # If we are stopping we will no longer schedule any jobs despite polling.
+        # We need to keep polling not to exceed the `max.poll.interval` for long-running
+        # non-blocking jobs and we need to allow them to finish. We however do not want to
+        # enqueue any new jobs. It's worth keeping in mind that it is the end user responsibility
+        # to detect shutdown in their long-running logic or else Karafka will force shutdown
+        # after a while.
+        #
+        # We do not care about resuming any partitions or lost jobs as we do not plan to do
+        # anything with them as we're in the shutdown phase.
+        wait_with_poll
+
+        # We do not want to schedule the shutdown jobs prior to finishing all the jobs
+        # (including non-blocking) as there might be a long-running job with a shutdown and then
+        # we would run two jobs in parallel for the same executor and consumer. We do not want that
+        # as it could create a race-condition.
+        build_and_schedule_shutdown_jobs
+
+        wait_with_poll
+
         shutdown
 
         # This is on purpose - see the notes for this method
@@ -100,57 +130,88 @@ module Karafka
 
       # Resumes processing of partitions that were paused due to an error.
       def resume_paused_partitions
-        @pauses_manager.resume { |topic, partition| @client.resume(topic, partition) }
+        @pauses_manager.resume do |topic, partition|
+          @client.resume(topic, partition)
+        end
       end
 
       # Enqueues revoking jobs for partitions that were taken away from the running process.
-      # @return [Boolean] was there anything to revoke
-      # @note We do not use scheduler here as those jobs are not meant to be order optimized in
-      #   any way. Since they operate occasionally it is irrelevant.
-      def schedule_revoke_lost_partitions_jobs
+      def build_and_schedule_revoke_lost_partitions_jobs
         revoked_partitions = @client.rebalance_manager.revoked_partitions
 
-        return false if revoked_partitions.empty?
+        # Stop early to save on some execution and array allocation
+        return if revoked_partitions.empty?
+
+        jobs = []
 
         revoked_partitions.each do |topic, partitions|
           partitions.each do |partition|
             pause_tracker = @pauses_manager.fetch(topic, partition)
             executor = @executors.fetch(topic, partition, pause_tracker)
-            @jobs_queue << Processing::Jobs::Revoked.new(executor)
+            jobs << Processing::Jobs::Revoked.new(executor)
           end
         end
 
-        true
+        @scheduler.schedule_revocation(@jobs_queue, jobs)
       end
 
-      # Takes the messages per topic partition and enqueues processing jobs in threads.
+      # Enqueues the shutdown jobs for all the executors that exist in our subscription group
+      def build_and_schedule_shutdown_jobs
+        jobs = []
+
+        @executors.each do |_, _, executor|
+          jobs << Processing::Jobs::Shutdown.new(executor)
+        end
+
+        @scheduler.schedule_shutdown(@jobs_queue, jobs)
+      end
+
+      # Polls messages within the time and amount boundaries defined in the settings and then
+      # builds karafka messages based on the raw rdkafka messages buffer returned by the
+      # `#batch_poll` method.
       #
-      # @param messages_buffer [Karafka::Connection::MessagesBuffer] buffer with messages
-      def schedule_partitions_jobs(messages_buffer)
-        @scheduler.call(messages_buffer) do |topic, partition, messages|
+      # @note There are two buffers, one for raw messages and one for "built" karafka messages
+      def poll_and_remap_messages
+        @messages_buffer.remap(
+          @client.batch_poll
+        )
+      end
+
+      # Takes the messages per topic partition and enqueues processing jobs in threads using
+      # given scheduler.
+      def build_and_schedule_consumption_jobs
+        return if @messages_buffer.empty?
+
+        jobs = []
+
+        @messages_buffer.each do |topic, partition, messages|
           pause = @pauses_manager.fetch(topic, partition)
 
           next if pause.paused?
 
           executor = @executors.fetch(topic, partition, pause)
 
-          @jobs_queue << Processing::Jobs::Consume.new(executor, messages)
+          jobs << Processing::Jobs::Consume.new(executor, messages)
         end
+
+        @scheduler.schedule_consumption(@jobs_queue, jobs)
       end
 
       # Waits for all the jobs from a given subscription group to finish before moving forward
-      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
-      def wait(subscription_group)
-        @jobs_queue.wait(subscription_group.id)
+      def wait
+        @jobs_queue.wait(@subscription_group.id)
+      end
+
+      # Waits without blocking the polling
+      # This should be used only when we no longer plan to use any incoming data and we can safely
+      # discard it
+      def wait_with_poll
+        @client.batch_poll until @jobs_queue.empty?(@subscription_group.id)
       end
 
       # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
       # stops kafka client.
       def shutdown
-        @jobs_queue.close
-        # This runs synchronously, making sure we finish all the shutdowns before we stop the
-        # client.
-        @executors.shutdown
         @client.commit_offsets!
         @client.stop
       end

data/lib/karafka/connection/listeners_batch.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Abstraction layer around listeners batch.
+    class ListenersBatch
+      include Enumerable
+
+      # @param jobs_queue [JobsQueue]
+      # @return [ListenersBatch]
+      def initialize(jobs_queue)
+        @batch = App.subscription_groups.map do |subscription_group|
+          Connection::Listener.new(subscription_group, jobs_queue)
+        end
+      end
+
+      # Iterates over available listeners and yields each listener
+      # @param block [Proc] block we want to run
+      def each(&block)
+        @batch.each(&block)
+      end
+    end
+  end
+end

data/lib/karafka/connection/messages_buffer.rb

@@ -2,20 +2,26 @@
 
 module Karafka
   module Connection
-    # Buffer for messages.
-    # When message is added to this buffer, it gets assigned to an array with other messages from
-    # the same topic and partition.
+    # Buffer used to build and store karafka messages built based on raw librdkafka messages.
     #
-    # @note This buffer is NOT threadsafe.
+    # Why do we have two buffers? `RawMessagesBuffer` is used to store raw messages and to handle
+    #   cases related to partition revocation and reconnections. It is "internal" to the listening
+    #   process. `MessagesBuffer` on the other hand is used to "translate" those raw messages that
+    #   we know that are ok into Karafka messages and to simplify further work with them.
+    #
+    # While it adds a bit of overhead, it makes conceptual things much easier and it adds only two
+    #   simple hash iterations over messages batch.
+    #
+    # @note This buffer is NOT thread safe. We do not worry about it as we do not use it outside
+    #   of the main listener loop. It can be cleared after the jobs are scheduled with messages
+    #   it stores, because messages arrays are not "cleared" in any way directly and their
+    #   reference stays.
     class MessagesBuffer
       attr_reader :size
 
-      extend Forwardable
-
-      def_delegators :@groups, :each
-
-      # @return [Karafka::Connection::MessagesBuffer] buffer instance
-      def initialize
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
+      def initialize(subscription_group)
+        @subscription_group = subscription_group
         @size = 0
         @groups = Hash.new do |topic_groups, topic|
           topic_groups[topic] = Hash.new do |partition_groups, partition|
@@ -24,64 +30,54 @@ module Karafka
         end
       end
 
-      # Adds a message to the buffer.
-      #
-      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
-      # @return [Array<Rdkafka::Consumer::Message>] given partition topic sub-buffer array
-      def <<(message)
-        @size += 1
-        @groups[message.topic][message.partition] << message
-      end
+      # Remaps raw messages from the raw messages buffer to Karafka messages
+      # @param raw_messages_buffer [RawMessagesBuffer] buffer with raw messages
+      def remap(raw_messages_buffer)
+        clear unless @size.zero?
 
-      # Removes given topic and partition data out of the buffer
-      # This is used when there's a partition revocation
-      # @param topic [String] topic we're interested in
-      # @param partition [Integer] partition of which data we want to remove
-      def delete(topic, partition)
-        return unless @groups.key?(topic)
-        return unless @groups.fetch(topic).key?(partition)
+        # Since it happens "right after" we've received the messages, it is close enough it time
+        # to be used as the moment we received messages.
+        received_at = Time.now
 
-        topic_data = @groups.fetch(topic)
-        topic_data.delete(partition)
+        raw_messages_buffer.each do |topic, partition, messages|
+          @size += messages.count
 
-        recount!
+          ktopic = @subscription_group.topics.find(topic)
 
-        # If there are no more partitions to handle in a given topic, remove it completely
-        @groups.delete(topic) if topic_data.empty?
+          @groups[topic][partition] = messages.map do |message|
+            Messages::Builders::Message.call(
+              message,
+              ktopic,
+              received_at
+            )
+          end
+        end
       end
 
-      # Removes duplicated messages from the same partitions
-      # This should be used only when rebalance occurs, as we may get data again we already have
-      # due to the processing from the last offset. In cases like this, we may get same data
-      # again and we do want to ensure as few duplications as possible
-      def uniq!
-        @groups.each_value do |partitions|
-          partitions.each_value do |messages|
-            messages.uniq!(&:offset)
+      # Allows to iterate over all the topics and partitions messages
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Array<Karafka::Messages::Message>] messages from a given topic partition
+      def each
+        @groups.each do |topic, partitions|
+          partitions.each do |partition, messages|
+            yield(topic, partition, messages)
           end
         end
-
-        recount!
       end
 
-      # Removes all the data from the buffer.
-      #
-      # @note We do not clear the whole groups hash but rather we clear the partition hashes, so
-      #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
-      #   may be used in other threads for data processing, thus if we would clear it, we could
-      #   potentially clear a raw messages array for a job that is in the jobs queue.
-      def clear
-        @size = 0
-        @groups.each_value(&:clear)
+      # @return [Boolean] is the buffer empty or does it contain any messages
+      def empty?
+        @size.zero?
       end
 
       private
 
-      # Updates the messages count if we performed any operations that could change the state
-      def recount!
-        @size = @groups.each_value.sum do |partitions|
-          partitions.each_value.map(&:count).sum
-        end
+      # Clears the buffer completely
+      def clear
+        @size = 0
+        @groups.clear
       end
     end
   end

data/lib/karafka/connection/raw_messages_buffer.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Buffer for raw librdkafka messages.
+    #
+    # When message is added to this buffer, it gets assigned to an array with other messages from
+    # the same topic and partition.
+    #
+    # @note This buffer is NOT threadsafe.
+    #
+    # @note We store data here in groups per topic partition to handle the revocation case, where
+    #    we may need to remove messages from a single topic partition.
+    class RawMessagesBuffer
+      attr_reader :size
+
+      # @return [Karafka::Connection::MessagesBuffer] buffer instance
+      def initialize
+        @size = 0
+        @groups = Hash.new do |topic_groups, topic|
+          topic_groups[topic] = Hash.new do |partition_groups, partition|
+            partition_groups[partition] = []
+          end
+        end
+      end
+
+      # Adds a message to the buffer.
+      #
+      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
+      # @return [Array<Rdkafka::Consumer::Message>] given partition topic sub-buffer array
+      def <<(message)
+        @size += 1
+        @groups[message.topic][message.partition] << message
+      end
+
+      # Allows to iterate over all the topics and partitions messages
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
+      def each
+        @groups.each do |topic, partitions|
+          partitions.each do |partition, messages|
+            yield(topic, partition, messages)
+          end
+        end
+      end
+
+      # Removes given topic and partition data out of the buffer
+      # This is used when there's a partition revocation
+      # @param topic [String] topic we're interested in
+      # @param partition [Integer] partition of which data we want to remove
+      def delete(topic, partition)
+        return unless @groups.key?(topic)
+        return unless @groups.fetch(topic).key?(partition)
+
+        topic_data = @groups.fetch(topic)
+        topic_data.delete(partition)
+
+        recount!
+
+        # If there are no more partitions to handle in a given topic, remove it completely
+        @groups.delete(topic) if topic_data.empty?
+      end
+
+      # Removes duplicated messages from the same partitions
+      # This should be used only when rebalance occurs, as we may get data again we already have
+      # due to the processing from the last offset. In cases like this, we may get same data
+      # again and we do want to ensure as few duplications as possible
+      def uniq!
+        @groups.each_value do |partitions|
+          partitions.each_value do |messages|
+            messages.uniq!(&:offset)
+          end
+        end
+
+        recount!
+      end
+
+      # Removes all the data from the buffer.
+      #
+      # @note We do not clear the whole groups hash but rather we clear the partition hashes, so
+      #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
+      #   may be used in other threads for data processing, thus if we would clear it, we could
+      #   potentially clear a raw messages array for a job that is in the jobs queue.
+      def clear
+        @size = 0
+        @groups.each_value(&:clear)
+      end
+
+      private
+
+      # Updates the messages count if we performed any operations that could change the state
+      def recount!
+        @size = @groups.each_value.sum do |partitions|
+          partitions.each_value.map(&:count).sum
+        end
+      end
+    end
+  end
+end

data/lib/karafka/contracts/config.rb

@@ -25,6 +25,7 @@ module Karafka
         required(:pause_max_timeout) { int? & gt?(0) }
         required(:pause_with_exponential_backoff).filled(:bool?)
         required(:shutdown_timeout) { int? & gt?(0) }
+        required(:max_wait_time) { int? & gt?(0) }
         required(:kafka).filled(:hash)
 
         # We validate internals just to be sure, that they are present and working
@@ -53,6 +54,12 @@ module Karafka
           key(:pause_timeout).failure(:max_timeout_vs_pause_max_timeout)
         end
       end
+
+      rule(:shutdown_timeout, :max_wait_time) do
+        if values[:max_wait_time].to_i >= values[:shutdown_timeout].to_i
+          key(:shutdown_timeout).failure(:shutdown_timeout_vs_max_wait_time)
+        end
+      end
     end
   end
 end

data/lib/karafka/helpers/async.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Allows a given class to run async in a separate thread. Provides also few methods we may
+    # want to use to control the underlying thread
+    #
+    # @note Thread running code needs to manage it's own exceptions. If they leak out, they will
+    #   abort thread on exception.
+    module Async
+      class << self
+        # Adds forwardable to redirect thread-based control methods to the underlying thread that
+        # runs the async operations
+        #
+        # @param base [Class] class we're including this module in
+        def included(base)
+          base.extend ::Forwardable
+
+          base.def_delegators :@thread, :join, :terminate, :alive?
+        end
+      end
+
+      # Runs the `#call` method in a new thread
+      def async_call
+        @thread = Thread.new do
+          Thread.current.abort_on_exception = true
+
+          call
+        end
+      end
+    end
+  end
+end

data/lib/karafka/messages/batch_metadata.rb

@@ -13,10 +13,33 @@ module Karafka
       :deserializer,
       :partition,
       :topic,
+      :created_at,
       :scheduled_at,
-      :consumption_lag,
-      :processing_lag,
+      :processed_at,
       keyword_init: true
-    )
+    ) do
+      # This lag describes how long did it take for a message to be consumed from the moment it was
+      # created
+      def consumption_lag
+        time_distance_in_ms(processed_at, created_at)
+      end
+
+      # This lag describes how long did a batch have to wait before it was picked up by one of the
+      # workers
+      def processing_lag
+        time_distance_in_ms(processed_at, scheduled_at)
+      end
+
+      private
+
+      # Computes time distance in between two times in ms
+      #
+      # @param time1 [Time]
+      # @param time2 [Time]
+      # @return [Integer] distance in between two times in ms
+      def time_distance_in_ms(time1, time2)
+        ((time1 - time2) * 1_000).round
+      end
+    end
   end
 end