karafka-core 2.0.0

data/lib/karafka/core/configurable.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # A simple dry-configuration API compatible module for defining settings with defaults and a
+    # constructor.
+    module Configurable
+      # A simple settings layer that works similar to dry-configurable
+      # It allows us to define settings on a class and per instance level with templating on a class
+      # level. It handles inheritance and allows for nested settings.
+      #
+      # @note The core settings template needs to be defined on a class level
+      class << self
+        # Sets up all the class methods and inits the core root node.
+        # Useful when only per class settings are needed as does not include instance methods
+        # @param base [Class] class that we extend
+        def extended(base)
+          base.extend ClassMethods
+        end
+
+        # Sets up all the class and instance methods and inits the core root node
+        #
+        # @param base [Class] class to which we want to add configuration
+        #
+        # Needs to be used when per instance configuration is needed
+        def included(base)
+          base.include InstanceMethods
+          base.extend self
+        end
+      end
+
+      # Instance related methods
+      module InstanceMethods
+        # @return [Node] config root node
+        def config
+          @config ||= self.class.config.deep_dup
+        end
+
+        # Allows for a per instance configuration (if needed)
+        # @param block [Proc] block for configuration
+        def configure(&block)
+          config.configure(&block)
+        end
+      end
+
+      # Class related methods
+      module ClassMethods
+        # @return [Node] root node for the settings
+        def config
+          return @config if @config
+
+          # This will handle inheritance
+          @config = if superclass.respond_to?(:config)
+                      superclass.config.deep_dup
+                    else
+                      Node.new(:root)
+                    end
+        end
+
+        # Allows for a per class configuration (if needed)
+        # @param block [Proc] block for configuration
+        def configure(&block)
+          config.configure(&block)
+        end
+
+        # Pipes the settings setup to the config root node
+        def setting(...)
+          config.setting(...)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/contractable/contract.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    module Contractable
+      # Base contract for all the contracts that check data format
+      #
+      # @note This contract does NOT support rules inheritance as it was never needed in Karafka
+      class Contract
+        extend Core::Configurable
+
+        # Yaml based error messages data
+        setting(:error_messages)
+
+        # Class level API definitions
+        class << self
+          # @return [Array<Rule>] all the validation rules defined for a given contract
+          attr_reader :rules
+
+          # Allows for definition of a scope/namespace for nested validations
+          #
+          # @param path [Symbol] path in the hash for nesting
+          # @param block [Proc] nested rule code or more nestings inside
+          #
+          # @example
+          #   nested(:key) do
+          #     required(:inside) { |inside| inside.is_a?(String) }
+          #   end
+          def nested(path, &block)
+            init_accu
+            @nested << path
+            instance_eval(&block)
+            @nested.pop
+          end
+
+          # Defines a rule for a required field (required means, that will automatically create an
+          # error if missing)
+          #
+          # @param keys [Array<Symbol>] single or full path
+          # @param block [Proc] validation rule
+          def required(*keys, &block)
+            init_accu
+            @rules << Rule.new(@nested + keys, :required, block).freeze
+          end
+
+          # @param keys [Array<Symbol>] single or full path
+          # @param block [Proc] validation rule
+          def optional(*keys, &block)
+            init_accu
+            @rules << Rule.new(@nested + keys, :optional, block).freeze
+          end
+
+          # @param block [Proc] validation rule
+          #
+          # @note Virtual rules have different result expectations. Please see contracts or specs for
+          #   details.
+          def virtual(&block)
+            init_accu
+            @rules << Rule.new([], :virtual, block).freeze
+          end
+
+          private
+
+          # Initializes nestings and rules building accumulator
+          def init_accu
+            @nested ||= []
+            @rules ||= []
+          end
+        end
+
+        # Runs the validation
+        #
+        # @param data [Hash] hash with data we want to validate
+        # @return [Result] validaton result
+        def call(data)
+          errors = []
+
+          self.class.rules.map do |rule|
+            case rule.type
+            when :required
+              validate_required(data, rule, errors)
+            when :optional
+              validate_optional(data, rule, errors)
+            when :virtual
+              validate_virtual(data, rule, errors)
+            end
+          end
+
+          Result.new(errors, self)
+        end
+
+        # @param data [Hash] data for validation
+        # @param error_class [Class] error class that should be used when validation fails
+        # @return [Boolean] true
+        # @raise [StandardError] any error provided in the error_class that inherits from the
+        #   standard error
+        def validate!(data, error_class)
+          result = call(data)
+
+          return true if result.success?
+
+          raise error_class, result.errors
+        end
+
+        private
+
+        # Runs validation for rules on fields that are required and adds errors (if any) to the
+        # errors array
+        #
+        # @param data [Hash] input hash
+        # @param rule [Rule] validation rule
+        # @param errors [Array] array with errors from previous rules (if any)
+        def validate_required(data, rule, errors)
+          for_checking = dig(data, rule.path)
+
+          if for_checking.first == :match
+            result = rule.validator.call(for_checking.last, data, errors, self)
+
+            return if result == true
+
+            errors << [rule.path, result || :format]
+          else
+            errors << [rule.path, :missing]
+          end
+        end
+
+        # Runs validation for rules on fields that are optional and adds errors (if any) to the
+        # errors array
+        #
+        # @param data [Hash] input hash
+        # @param rule [Rule] validation rule
+        # @param errors [Array] array with errors from previous rules (if any)
+        def validate_optional(data, rule, errors)
+          for_checking = dig(data, rule.path)
+
+          return unless for_checking.first == :match
+
+          result = rule.validator.call(for_checking.last, data, errors, self)
+
+          return if result == true
+
+          errors << [rule.path, result || :format]
+        end
+
+        # Runs validation for rules on virtual fields (aggregates, etc) and adds errors (if any) to
+        # the errors array
+        #
+        # @param data [Hash] input hash
+        # @param rule [Rule] validation rule
+        # @param errors [Array] array with errors from previous rules (if any)
+        def validate_virtual(data, rule, errors)
+          result = rule.validator.call(data, errors, self)
+
+          return if result == true
+
+          errors.push(*result)
+        end
+
+        # Tries to dig for a given key in a hash and returns it with indication whether or not it was
+        # possible to find it (dig returns nil and we don't know if it wasn't the digged key value)
+        #
+        # @param data [Hash]
+        # @param keys [Array<Symbol>]
+        # @return [Array<Symbol, Object>] array where the first element is `:match` or `:miss` and
+        #   the digged value or nil if not found
+        def dig(data, keys)
+          current = data
+          result = :match
+
+          keys.each do |nesting|
+            unless current.key?(nesting)
+              result = :miss
+
+              break
+            end
+
+            current = current[nesting]
+          end
+
+          [result, current]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/contractable/result.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    module Contractable
+      # Representation of a validaton result with resolved error messages
+      class Result
+        attr_reader :errors
+
+        # Builds a result object and remaps (if needed) error keys to proper error messages
+        #
+        # @param errors [Array<Array>] array with sub-arrays with paths and error keys
+        # @param contract [Object] contract that generated the error
+        def initialize(errors, contract)
+          # Short track to skip object allocation for the happy path
+          if errors.empty?
+            @errors = errors
+            return
+          end
+
+          hashed = {}
+
+          errors.each do |error|
+            scope = error.first.map(&:to_s).join('.').to_sym
+
+            # This will allow for usage of custom messages instead of yaml keys if needed
+            hashed[scope] = if error.last.is_a?(String)
+                              error.last
+                            else
+                              build_message(contract, scope, error.last)
+                            end
+          end
+
+          @errors = hashed
+        end
+
+        # @return [Boolean] true if no errors
+        def success?
+          errors.empty?
+        end
+
+        private
+
+        # Builds message based on the error messages
+        # @param contract [Object] contract for which we build the result
+        # @param scope [Symbol] path to the key that has an error
+        # @param error_key [Symbol] error key for yaml errors lookup
+        # @return [String] error message
+        def build_message(contract, scope, error_key)
+          messages = contract.class.config.error_messages
+
+          messages.fetch(error_key.to_s) do
+            messages.fetch("#{scope}_#{error_key}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/contractable/rule.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    module Contractable
+      # Representation of a single validation rule
+      Rule = Struct.new(:path, :type, :validator)
+    end
+  end
+end

data/lib/karafka/core/contractable.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # Contract layer for the Karafka ecosystem
+    # It aims to be "dry-validation" like but smaller and easier to handle + without dependencies
+    #
+    # It allows for nested validations, etc
+    #
+    # @note It is thread-safe to run but validations definitions should happen before threads are
+    #   used.
+    module Contractable
+    end
+  end
+end

data/lib/karafka/core/monitoring/event.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    module Monitoring
+      # Single notification event wrapping payload with id
+      class Event
+        attr_reader :id, :payload
+
+        # @param id [String, Symbol] id of the event
+        # @param payload [Hash] event payload
+        def initialize(id, payload)
+          @id = id
+          @payload = payload
+        end
+
+        # Hash access to the payload data (if present)
+        #
+        # @param [String, Symbol] name
+        def [](name)
+          @payload.fetch(name)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/monitoring/monitor.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    module Monitoring
+      # Karafka monitor that can be used to pass through instrumentation calls to selected
+      # notifications bus.
+      #
+      # It provides abstraction layer that allows us to use both our internal notifications as well
+      # as `ActiveSupport::Notifications`.
+      class Monitor
+        # Empty has to save on objects allocation
+        EMPTY_HASH = {}.freeze
+
+        private_constant :EMPTY_HASH
+
+        # @param notifications_bus [Object] either our internal notifications bus or
+        #   `ActiveSupport::Notifications`
+        # @param namespace [String, nil] namespace for events or nil if no namespace
+        def initialize(notifications_bus, namespace = nil)
+          @notifications_bus = notifications_bus
+          @namespace = namespace
+          @mapped_events = Concurrent::Map.new
+        end
+
+        # Passes the instrumentation block (if any) into the notifications bus
+        #
+        # @param event_id [String, Symbol] event id
+        # @param payload [Hash]
+        # @param block [Proc] block we want to instrument (if any)
+        def instrument(event_id, payload = EMPTY_HASH, &block)
+          full_event_name = @mapped_events[event_id] ||= [event_id, @namespace].compact.join('.')
+
+          @notifications_bus.instrument(full_event_name, payload, &block)
+        end
+
+        # Allows us to subscribe to the notification bus
+        #
+        # @param args [Array] any arguments that the notification bus subscription layer accepts
+        # @param block [Proc] optional block for subscription
+        def subscribe(*args, &block)
+          @notifications_bus.subscribe(*args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/monitoring/notifications.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    module Monitoring
+      # A simple notifications layer for Karafka ecosystem that aims to provide API compatible
+      # with both `ActiveSupport::Notifications` and `dry-monitor`.
+      #
+      # We do not use any of them by default as our use-case is fairly simple and we do not want
+      # to have too many external dependencies.
+      class Notifications
+        attr_reader :name
+
+        # Raised when someone wants to publish event that was not registered
+        EventNotRegistered = Class.new(StandardError)
+
+        # Empty hash for internal referencing
+        EMPTY_HASH = {}.freeze
+
+        private_constant :EMPTY_HASH
+
+        def initialize
+          @listeners = Concurrent::Map.new { |k, v| k[v] = Concurrent::Array.new }
+          # This allows us to optimize the method calling lookups
+          @events_methods_map = Concurrent::Map.new
+        end
+
+        # Registers a new event on which we can publish
+        #
+        # @param event_id [String, Symbol] event id
+        def register_event(event_id)
+          @listeners[event_id]
+          @events_methods_map[event_id] = :"on_#{event_id.to_s.tr('.', '_')}"
+        end
+
+        # Clears all the subscribed listeners
+        def clear
+          @listeners.each_value(&:clear)
+        end
+
+        # Allows for subscription to an event
+        # There are two ways you can subscribe: via block or via listener.
+        #
+        # @param event_id_or_listener [Object] event id when we want to subscribe to a particular
+        #   event with a block or listener if we want to subscribe with general listener
+        # @param block [Proc] block of code if we want to subscribe with it
+        #
+        # @example Subscribe using listener
+        #   subscribe(MyListener.new)
+        #
+        # @example Subscribe via block
+        #   subscribe do |event|
+        #     puts event
+        #   end
+        def subscribe(event_id_or_listener, &block)
+          if block
+            event_id = event_id_or_listener
+
+            raise EventNotRegistered, event_id unless @listeners.key?(event_id)
+
+            @listeners[event_id] << block
+          else
+            listener = event_id_or_listener
+
+            @listeners.each_key do |reg_event_id|
+              next unless listener.respond_to?(@events_methods_map[reg_event_id])
+
+              @listeners[reg_event_id] << listener
+            end
+          end
+        end
+
+        # Allows for code instrumentation
+        # Runs the provided code and sends the instrumentation details to all registered listeners
+        #
+        # @param event_id [String, Symbol] id of the event
+        # @param payload [Hash] payload for the instrumentation
+        # @param block [Proc] instrumented code
+        # @return [Object] whatever the provided block (if any) returns
+        #
+        # @example Instrument some code
+        #   instrument('sleeping') do
+        #     sleep(1)
+        #   end
+        def instrument(event_id, payload = EMPTY_HASH, &block)
+          result, time = measure_time_taken(&block) if block_given?
+
+          event = Event.new(
+            event_id,
+            time ? payload.merge(time: time) : payload
+          )
+
+          @listeners[event_id].each do |listener|
+            if listener.is_a?(Proc)
+              listener.call(event)
+            else
+              listener.send(@events_methods_map[event_id], event)
+            end
+          end
+
+          result
+        end
+
+        private
+
+        # Measures time taken to execute a given block and returns it together with the result of
+        # the block execution
+        def measure_time_taken
+          start = current_time
+          result = yield
+          [result, current_time - start]
+        end
+
+        # @return [Integer] current monotonic time
+        def current_time
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/monitoring.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Main module namespace
+module Karafka
+  module Core
+    # Monitoring for Karafka and WaterDrop
+    # It allows us to have a layer that can work with `dry-monitor` as well as
+    # `ActiveSupport::Notifications` or standalone depending on the case. Thanks to that we do not
+    # have to rely on third party tools that could break.
+    module Monitoring
+    end
+  end
+end

data/lib/karafka/core/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # Current Karafka::Core version
+    # We follow the versioning schema of given Karafka version
+    VERSION = '2.0.0'
+  end
+end

data/lib/karafka/core.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for small support modules used throughout the Karafka ecosystem
+  module Core
+    class << self
+      # @return [String] root path of this gem
+      def gem_root
+        Pathname.new(File.expand_path('../..', __dir__))
+      end
+    end
+  end
+end

data/lib/karafka-core.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+%w[
+  yaml
+  concurrent/map
+  concurrent/hash
+  concurrent/array
+  karafka/core
+  karafka/core/version
+  karafka/core/monitoring
+  karafka/core/monitoring/event
+  karafka/core/monitoring/monitor
+  karafka/core/monitoring/notifications
+  karafka/core/configurable
+  karafka/core/configurable/leaf
+  karafka/core/configurable/node
+  karafka/core/contractable/contract
+  karafka/core/contractable/result
+  karafka/core/contractable/rule
+].each { |dependency| require dependency }
+
+# Karafka framework main namespace
+module Karafka
+end

data.tar.gz.sig

@@ -0,0 +1 @@
+�nnDV��c�"TP��Wo�5���"�
�T3��ѕ��������U���ye��)�Pt�