karafka-web 0.10.4 → 0.11.0.beta1

data/lib/karafka/web/pro/commanding/commands/partitions/resume.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Commands
+          module Partitions
+            # Delegates the resuming of the processing
+            class Resume < Base
+              self.name = 'partitions.resume'
+
+              # Dispatches the seek request into the appropriate filter and indicates that the
+              # seeking is in an acceptance state
+              def call
+                Handlers::Partitions::Tracker.instance << command
+
+                # Publish back info on who did this with all the details for inspection
+                acceptance(command.to_h)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/commands/partitions/seek.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Commands
+          module Partitions
+            # Topic partition seek command request handler
+            class Seek < Base
+              self.name = 'partitions.seek'
+
+              # Dispatches the seek request into the appropriate filter and indicates that the
+              # seeking is in an acceptance state
+              def call
+                Handlers::Partitions::Tracker.instance << command
+
+                # Publish back info on who did this with all the details for inspection
+                acceptance(command.to_h)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web
@@ -43,6 +35,10 @@ module Karafka
             'auto.offset.reset': 'latest'
           }
 
+          setting :listeners, default: [
+            Handlers::Partitions::Listener.new
+          ]
+
           configure
         end
       end

data/lib/karafka/web/pro/commanding/contracts/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web

data/lib/karafka/web/pro/commanding/dispatcher.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web
@@ -22,23 +14,24 @@ module Karafka
         # Dispatcher requires Web UI to have a producer
         class Dispatcher
           # What schema do we have in current Karafka version for commands dispatches
-          SCHEMA_VERSION = '1.0.0'
+          SCHEMA_VERSION = '1.1.0'
 
           class << self
             # Dispatches the command request
             #
-            # @param name [String, Symbol] name of the command we want to deal with in the process
+            # @param command_name [String, Symbol] name of the command we want to deal with in the process
             # @param process_id [String] id of the process. We use name instead of id only
             #   because in the web ui we work with the full name and it is easier. Since
-            def command(name, process_id)
+            # @param params [Hash] hash with extra command params that some commands may use.
+            def request(command_name, process_id, params = {})
               produce(
                 process_id,
+                'request',
                 {
                   schema_version: SCHEMA_VERSION,
-                  type: 'command',
-                  command: {
-                    name: name
-                  },
+                  type: 'request',
+                  # Name is auto-generated and required. Should not be changed
+                  command: params.merge(name: command_name),
                   dispatched_at: Time.now.to_f,
                   process: {
                     id: process_id
@@ -47,14 +40,38 @@ module Karafka
               )
             end
 
-            # Dispatches the result request
+            # Dispatches the acceptance info. Indicates that a command was received and appropriate
+            # action will be taken but async. Useful for commands that may not take immediate
+            # actions upon receiving a command.
             #
-            # @param result [Object] anything that can be the result of the command execution
+            # @param command_name [String, Symbol] command that triggered this result
             # @param process_id [String] related process id
+            # @param params [Hash] input command params (or empty hash if none)
+            def acceptance(command_name, process_id, params = {})
+              produce(
+                process_id,
+                'acceptance',
+                {
+                  schema_version: SCHEMA_VERSION,
+                  type: 'acceptance',
+                  command: params.merge(name: command_name),
+                  dispatched_at: Time.now.to_f,
+                  process: {
+                    id: process_id
+                  }
+                }
+              )
+            end
+
+            # Dispatches the result request
+            #
             # @param command_name [String, Symbol] command that triggered this result
-            def result(result, process_id, command_name)
+            # @param process_id [String] related process id
+            # @param result [Object] anything that can be the result of the command execution
+            def result(command_name, process_id, result)
               produce(
                 process_id,
+                'result',
                 {
                   schema_version: SCHEMA_VERSION,
                   type: 'result',
@@ -79,20 +96,24 @@ module Karafka
 
             # @return [String] consumers commands topic
             def commands_topic
-              ::Karafka::Web.config.topics.consumers.commands
+              ::Karafka::Web.config.topics.consumers.commands.name
             end
 
             # Converts payload to json, compresses it and dispatches to Kafka
             #
-            # @param payload [Hash] hash with payload
             # @param process_id [String]
-            def produce(process_id, payload)
+            # @param type [String] type of the request
+            # @param payload [Hash] hash with payload
+            def produce(process_id, type, payload)
               producer.produce_async(
                 topic: commands_topic,
                 key: process_id,
                 partition: 0,
                 payload: ::Zlib::Deflate.deflate(payload.to_json),
-                headers: { 'zlib' => 'true' }
+                headers: {
+                  'zlib' => 'true',
+                  'type' => type
+                }
               )
             end
           end

data/lib/karafka/web/pro/commanding/handlers/partitions/commands/base.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          # Namespace for commands related to specific partitions
+          module Partitions
+            # Namespace for partitions related commands
+            module Commands
+              # Base class for all the partition related commands handlers
+              class Base
+                # @param listener [Karafka::Connection::Listener] listener that handles given
+                #   partition in the context of given subscription group
+                # @param client [Karafka::Connection::Client] underlying Kafka client
+                # @param request [Request] command request
+                def initialize(listener, client, request)
+                  @listener = listener
+                  @client = client
+                  @request = request
+                end
+
+                # Runs the command
+                def call
+                  raise NotImplementedError, 'Implement in a subclass'
+                end
+
+                private
+
+                attr_reader :listener, :client, :request
+
+                # @return [String] name of the topic on which the command should be applied
+                def topic
+                  @topic ||= request[:topic]
+                end
+
+                # @return [String] partition for which the command should be applied
+                def partition_id
+                  @partition_id ||= request[:partition_id]
+                end
+
+                # @return [Karafka::Processing::Coordinator, Karafka::Pro::Processing::Coordinator]
+                def coordinator
+                  @coordinator ||= listener.coordinators.find_or_create(topic, partition_id)
+                end
+
+                # Publishes the execution result back to Kafka
+                # @param status [String] execution status
+                def result(status)
+                  Commanding::Dispatcher.result(
+                    request.name,
+                    ::Karafka::Web.config.tracking.consumers.sampler.process_id,
+                    request.to_h.merge(status: status)
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/partitions/commands/pause.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Partitions
+            module Commands
+              # Executes the pause request to pause (or prolong) partition
+              class Pause < Base
+                # 10 years in ms
+                FOREVER_MS = 10 * 365 * 24 * 60 * 60 * 1000
+
+                private_constant :FOREVER_MS
+
+                # Triggers pausing of a given topic partition or updates the pause
+                def call
+                  # If pause is already there and we don't want to change the way it is, we skip
+                  if coordinator.pause_tracker.paused? && request[:prevent_override]
+                    result('prevented')
+
+                    return
+                  end
+
+                  duration = request[:duration]
+                  duration = FOREVER_MS if duration.zero?
+
+                  coordinator.pause_tracker.pause(duration)
+                  client.pause(topic, partition_id, nil, duration)
+
+                  result('applied')
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/partitions/commands/resume.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Partitions
+            module Commands
+              # Resumes paused partition
+              class Resume < Base
+                # Expires the pause so Karafka resumes processing of the given topic partition
+                def call
+                  coordinator.pause_tracker.expire
+                  coordinator.pause_tracker.reset if request[:reset_attempts]
+
+                  result('applied')
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/partitions/commands/seek.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Partitions
+            module Commands
+              # Moves the offset and optionally also resumes processing (if applicable) to where
+              # user wanted
+              class Seek < Base
+                # Runs seeking with some extra options if applicable
+                def call
+                  # If user enabled overtaking prevention and we're already ahead of the requested
+                  # offset, we should ditch such a request
+                  if prevent_overtaking? &&
+                     coordinator.seek_offset &&
+                     coordinator.seek_offset >= desired_offset
+                    result('prevented')
+
+                    return
+                  end
+
+                  # Mark previous offset as consumed. We move the offset in case the first message
+                  # after seeking would be a poison pill. That way the offset position is
+                  # moved even if we get a rebalance later.
+                  assigned = client.mark_as_consumed!(
+                    seek_message(desired_offset - 1)
+                  )
+
+                  # If we were not able to mark as consumed it means that the assignment was lost
+                  # We should signal this and stop
+                  unless assigned
+                    result('lost_partition')
+
+                    return
+                  end
+
+                  client.seek(seek_message(desired_offset))
+
+                  coordinator.seek_offset = desired_offset
+                  # Clear the attempts. Previous attempts should not count to a changed offset and
+                  # we should start with a clean slate. That's why we reset the tracker
+                  coordinator.pause_tracker.reset
+                  # If there was a pause and if user no longer wants to wait until it expires, we
+                  # can reset it so the work starts immediately.
+                  coordinator.pause_tracker.expire if force_resume?
+
+                  result('applied')
+                end
+
+                private
+
+                # @param offset [Integer] desired seek offset
+                # @return [::Karafka::Messages::Seek] builds a seek message for offset change
+                def seek_message(offset)
+                  ::Karafka::Messages::Seek.new(topic, partition_id, offset)
+                end
+
+                # @return [Integer] desired offset to move to
+                def desired_offset
+                  @desired_offset ||= request[:offset]
+                end
+
+                # @return [Boolean] does user want to use the overtaking prevention
+                def prevent_overtaking?
+                  request[:prevent_overtaking]
+                end
+
+                # @return [Boolean] should we resume immediately instead of waiting until the pause
+                #   timeout expires
+                def force_resume?
+                  request[:force_resume]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/partitions/executor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Namespace for handlers that deal with executing in async (outside of the request cycle)
+        # the requested commands.
+        module Handlers
+          # Namespace for handling requests related to consuming Kafka topics partitions
+          module Partitions
+            # Selects proper command for running prior to next poll and executes its flow
+            class Executor
+              # @param listener [Karafka::Connection::Listener]
+              # @param client [Karafka::Connection::Client]
+              # @param request [Request]
+              def call(listener, client, request)
+                command = case request.name
+                          when Commanding::Commands::Partitions::Resume.name
+                            Commands::Resume
+                          when Commanding::Commands::Partitions::Pause.name
+                            Commands::Pause
+                          when Commanding::Commands::Partitions::Seek.name
+                            Commands::Seek
+                          else
+                            raise ::Karafka::Errors::UnsupportedCaseError, request.name
+                          end
+
+                command.new(
+                  listener,
+                  client,
+                  request
+                ).call
+              end
+
+              # Publishes the reject event as the final result. Used to indicate, that given
+              # request will not be processed because it is not valid anymore.
+              #
+              # @param request [Request]
+              def reject(request)
+                Dispatcher.result(
+                  request.name,
+                  process_id,
+                  request.to_h.merge(status: 'rebalance_rejected')
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/partitions/listener.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Partitions
+            # Listener that hooks to the connection listener fetch loop flow to adjust it prior
+            # to the next pooling or on rebalances to execute the partition specific commands.
+            class Listener
+              def initialize
+                @tracker = Tracker.instance
+                @executor = Executor.new
+              end
+
+              # Connects to the fetching loop pre-fetch and executes requested commands before
+              # pooling (if any).
+              #
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_connection_listener_fetch_loop(event)
+                listener = event[:caller]
+                client = event[:client]
+
+                @tracker.each_for(listener.subscription_group.id) do |command|
+                  @executor.call(listener, client, command)
+                end
+              end
+
+              # Creates a rebalance barrier, so we do not execute any commands in between
+              # rebalances. This prevents us from aggregating old and outdated requests.
+              #
+              # Rejects all the commands if there were any waiting.
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_rebalance_partitions_assigned(event)
+                @tracker.each_for(event[:subscription_group_id]) do |command|
+                  @executor.reject(command)
+                end
+              end
+
+              # @param event [Karafka::Core::Monitoring::Event]
+              # @see `#on_rebalance_partitions_assigned`
+              def on_rebalance_partitions_revoked(event)
+                on_rebalance_partitions_assigned(event)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/partitions/tracker.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Partitions
+            # Tracker used to record incoming partition related operational requests until they are
+            # executable or invalid. It stores the requests as they come for execution pre-polling.
+            class Tracker
+              include Singleton
+
+              # Empty array for internal usage
+              EMPTY_ARRAY = [].freeze
+
+              private_constant :EMPTY_ARRAY
+
+              def initialize
+                @mutex = Mutex.new
+                @requests = Hash.new { |h, k| h[k] = [] }
+              end
+
+              # Adds the given command into the tracker so it can be retrieved when needed.
+              #
+              # @param command [Request] command we want to schedule
+              # @note We accumulate requests per subscription group because this is the layer of
+              #   applicability of those even for partition related requests.
+              def <<(command)
+                @mutex.synchronize do
+                  @requests[command[:subscription_group_id]] << command
+                end
+              end
+
+              # Selects all incoming command requests for given subscription group and iterates
+              # over them. It removes selected requests during iteration.
+              #
+              # @param subscription_group_id [String] id of the subscription group for which we
+              #   want to get all the requests. Subscription groups ids (not names) are unique
+              #   within the application, so it is unique "enough".
+              # @param block [Proc]
+              #
+              # @yieldparam [Request] given command request for the requested subscription group
+              def each_for(subscription_group_id, &block)
+                requests = nil
+
+                @mutex.synchronize do
+                  requests = @requests.delete(subscription_group_id)
+                end
+
+                (requests || EMPTY_ARRAY).each(&block)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/listener.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Web
@@ -34,7 +26,7 @@ module Karafka
             t_config = Karafka::Web.config.topics
 
             iterator = Karafka::Pro::Iterator.new(
-              { t_config.consumers.commands => true },
+              { t_config.consumers.commands.name => true },
               settings: c_config.kafka,
               yield_nil: true,
               max_wait_time: c_config.max_wait_time
@@ -87,7 +79,7 @@ module Karafka
               'error.occurred',
               error: error,
               caller: self,
-              type: 'web.controlling.controller.error'
+              type: 'web.commanding.listener.error'
             )
           end
         end