karafka-core 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ # A simple dry-configuration API compatible module for defining settings with defaults and a
6
+ # constructor.
7
+ module Configurable
8
+ # A simple settings layer that works similar to dry-configurable
9
+ # It allows us to define settings on a class and per instance level with templating on a class
10
+ # level. It handles inheritance and allows for nested settings.
11
+ #
12
+ # @note The core settings template needs to be defined on a class level
13
+ class << self
14
+ # Sets up all the class methods and inits the core root node.
15
+ # Useful when only per class settings are needed as does not include instance methods
16
+ # @param base [Class] class that we extend
17
+ def extended(base)
18
+ base.extend ClassMethods
19
+ end
20
+
21
+ # Sets up all the class and instance methods and inits the core root node
22
+ #
23
+ # @param base [Class] class to which we want to add configuration
24
+ #
25
+ # Needs to be used when per instance configuration is needed
26
+ def included(base)
27
+ base.include InstanceMethods
28
+ base.extend self
29
+ end
30
+ end
31
+
32
+ # Instance related methods
33
+ module InstanceMethods
34
+ # @return [Node] config root node
35
+ def config
36
+ @config ||= self.class.config.deep_dup
37
+ end
38
+
39
+ # Allows for a per instance configuration (if needed)
40
+ # @param block [Proc] block for configuration
41
+ def configure(&block)
42
+ config.configure(&block)
43
+ end
44
+ end
45
+
46
+ # Class related methods
47
+ module ClassMethods
48
+ # @return [Node] root node for the settings
49
+ def config
50
+ return @config if @config
51
+
52
+ # This will handle inheritance
53
+ @config = if superclass.respond_to?(:config)
54
+ superclass.config.deep_dup
55
+ else
56
+ Node.new(:root)
57
+ end
58
+ end
59
+
60
+ # Allows for a per class configuration (if needed)
61
+ # @param block [Proc] block for configuration
62
+ def configure(&block)
63
+ config.configure(&block)
64
+ end
65
+
66
+ # Pipes the settings setup to the config root node
67
+ def setting(...)
68
+ config.setting(...)
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,185 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ module Contractable
6
+ # Base contract for all the contracts that check data format
7
+ #
8
+ # @note This contract does NOT support rules inheritance as it was never needed in Karafka
9
+ class Contract
10
+ extend Core::Configurable
11
+
12
+ # Yaml based error messages data
13
+ setting(:error_messages)
14
+
15
+ # Class level API definitions
16
+ class << self
17
+ # @return [Array<Rule>] all the validation rules defined for a given contract
18
+ attr_reader :rules
19
+
20
+ # Allows for definition of a scope/namespace for nested validations
21
+ #
22
+ # @param path [Symbol] path in the hash for nesting
23
+ # @param block [Proc] nested rule code or more nestings inside
24
+ #
25
+ # @example
26
+ # nested(:key) do
27
+ # required(:inside) { |inside| inside.is_a?(String) }
28
+ # end
29
+ def nested(path, &block)
30
+ init_accu
31
+ @nested << path
32
+ instance_eval(&block)
33
+ @nested.pop
34
+ end
35
+
36
+ # Defines a rule for a required field (required means, that will automatically create an
37
+ # error if missing)
38
+ #
39
+ # @param keys [Array<Symbol>] single or full path
40
+ # @param block [Proc] validation rule
41
+ def required(*keys, &block)
42
+ init_accu
43
+ @rules << Rule.new(@nested + keys, :required, block).freeze
44
+ end
45
+
46
+ # @param keys [Array<Symbol>] single or full path
47
+ # @param block [Proc] validation rule
48
+ def optional(*keys, &block)
49
+ init_accu
50
+ @rules << Rule.new(@nested + keys, :optional, block).freeze
51
+ end
52
+
53
+ # @param block [Proc] validation rule
54
+ #
55
+ # @note Virtual rules have different result expectations. Please see contracts or specs for
56
+ # details.
57
+ def virtual(&block)
58
+ init_accu
59
+ @rules << Rule.new([], :virtual, block).freeze
60
+ end
61
+
62
+ private
63
+
64
+ # Initializes nestings and rules building accumulator
65
+ def init_accu
66
+ @nested ||= []
67
+ @rules ||= []
68
+ end
69
+ end
70
+
71
+ # Runs the validation
72
+ #
73
+ # @param data [Hash] hash with data we want to validate
74
+ # @return [Result] validaton result
75
+ def call(data)
76
+ errors = []
77
+
78
+ self.class.rules.map do |rule|
79
+ case rule.type
80
+ when :required
81
+ validate_required(data, rule, errors)
82
+ when :optional
83
+ validate_optional(data, rule, errors)
84
+ when :virtual
85
+ validate_virtual(data, rule, errors)
86
+ end
87
+ end
88
+
89
+ Result.new(errors, self)
90
+ end
91
+
92
+ # @param data [Hash] data for validation
93
+ # @param error_class [Class] error class that should be used when validation fails
94
+ # @return [Boolean] true
95
+ # @raise [StandardError] any error provided in the error_class that inherits from the
96
+ # standard error
97
+ def validate!(data, error_class)
98
+ result = call(data)
99
+
100
+ return true if result.success?
101
+
102
+ raise error_class, result.errors
103
+ end
104
+
105
+ private
106
+
107
+ # Runs validation for rules on fields that are required and adds errors (if any) to the
108
+ # errors array
109
+ #
110
+ # @param data [Hash] input hash
111
+ # @param rule [Rule] validation rule
112
+ # @param errors [Array] array with errors from previous rules (if any)
113
+ def validate_required(data, rule, errors)
114
+ for_checking = dig(data, rule.path)
115
+
116
+ if for_checking.first == :match
117
+ result = rule.validator.call(for_checking.last, data, errors, self)
118
+
119
+ return if result == true
120
+
121
+ errors << [rule.path, result || :format]
122
+ else
123
+ errors << [rule.path, :missing]
124
+ end
125
+ end
126
+
127
+ # Runs validation for rules on fields that are optional and adds errors (if any) to the
128
+ # errors array
129
+ #
130
+ # @param data [Hash] input hash
131
+ # @param rule [Rule] validation rule
132
+ # @param errors [Array] array with errors from previous rules (if any)
133
+ def validate_optional(data, rule, errors)
134
+ for_checking = dig(data, rule.path)
135
+
136
+ return unless for_checking.first == :match
137
+
138
+ result = rule.validator.call(for_checking.last, data, errors, self)
139
+
140
+ return if result == true
141
+
142
+ errors << [rule.path, result || :format]
143
+ end
144
+
145
+ # Runs validation for rules on virtual fields (aggregates, etc) and adds errors (if any) to
146
+ # the errors array
147
+ #
148
+ # @param data [Hash] input hash
149
+ # @param rule [Rule] validation rule
150
+ # @param errors [Array] array with errors from previous rules (if any)
151
+ def validate_virtual(data, rule, errors)
152
+ result = rule.validator.call(data, errors, self)
153
+
154
+ return if result == true
155
+
156
+ errors.push(*result)
157
+ end
158
+
159
+ # Tries to dig for a given key in a hash and returns it with indication whether or not it was
160
+ # possible to find it (dig returns nil and we don't know if it wasn't the digged key value)
161
+ #
162
+ # @param data [Hash]
163
+ # @param keys [Array<Symbol>]
164
+ # @return [Array<Symbol, Object>] array where the first element is `:match` or `:miss` and
165
+ # the digged value or nil if not found
166
+ def dig(data, keys)
167
+ current = data
168
+ result = :match
169
+
170
+ keys.each do |nesting|
171
+ unless current.key?(nesting)
172
+ result = :miss
173
+
174
+ break
175
+ end
176
+
177
+ current = current[nesting]
178
+ end
179
+
180
+ [result, current]
181
+ end
182
+ end
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ module Contractable
6
+ # Representation of a validaton result with resolved error messages
7
+ class Result
8
+ attr_reader :errors
9
+
10
+ # Builds a result object and remaps (if needed) error keys to proper error messages
11
+ #
12
+ # @param errors [Array<Array>] array with sub-arrays with paths and error keys
13
+ # @param contract [Object] contract that generated the error
14
+ def initialize(errors, contract)
15
+ # Short track to skip object allocation for the happy path
16
+ if errors.empty?
17
+ @errors = errors
18
+ return
19
+ end
20
+
21
+ hashed = {}
22
+
23
+ errors.each do |error|
24
+ scope = error.first.map(&:to_s).join('.').to_sym
25
+
26
+ # This will allow for usage of custom messages instead of yaml keys if needed
27
+ hashed[scope] = if error.last.is_a?(String)
28
+ error.last
29
+ else
30
+ build_message(contract, scope, error.last)
31
+ end
32
+ end
33
+
34
+ @errors = hashed
35
+ end
36
+
37
+ # @return [Boolean] true if no errors
38
+ def success?
39
+ errors.empty?
40
+ end
41
+
42
+ private
43
+
44
+ # Builds message based on the error messages
45
+ # @param contract [Object] contract for which we build the result
46
+ # @param scope [Symbol] path to the key that has an error
47
+ # @param error_key [Symbol] error key for yaml errors lookup
48
+ # @return [String] error message
49
+ def build_message(contract, scope, error_key)
50
+ messages = contract.class.config.error_messages
51
+
52
+ messages.fetch(error_key.to_s) do
53
+ messages.fetch("#{scope}_#{error_key}")
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ module Contractable
6
+ # Representation of a single validation rule
7
+ Rule = Struct.new(:path, :type, :validator)
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ # Contract layer for the Karafka ecosystem
6
+ # It aims to be "dry-validation" like but smaller and easier to handle + without dependencies
7
+ #
8
+ # It allows for nested validations, etc
9
+ #
10
+ # @note It is thread-safe to run but validations definitions should happen before threads are
11
+ # used.
12
+ module Contractable
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ module Monitoring
6
+ # Single notification event wrapping payload with id
7
+ class Event
8
+ attr_reader :id, :payload
9
+
10
+ # @param id [String, Symbol] id of the event
11
+ # @param payload [Hash] event payload
12
+ def initialize(id, payload)
13
+ @id = id
14
+ @payload = payload
15
+ end
16
+
17
+ # Hash access to the payload data (if present)
18
+ #
19
+ # @param [String, Symbol] name
20
+ def [](name)
21
+ @payload.fetch(name)
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ module Monitoring
6
+ # Karafka monitor that can be used to pass through instrumentation calls to selected
7
+ # notifications bus.
8
+ #
9
+ # It provides abstraction layer that allows us to use both our internal notifications as well
10
+ # as `ActiveSupport::Notifications`.
11
+ class Monitor
12
+ # Empty has to save on objects allocation
13
+ EMPTY_HASH = {}.freeze
14
+
15
+ private_constant :EMPTY_HASH
16
+
17
+ # @param notifications_bus [Object] either our internal notifications bus or
18
+ # `ActiveSupport::Notifications`
19
+ # @param namespace [String, nil] namespace for events or nil if no namespace
20
+ def initialize(notifications_bus, namespace = nil)
21
+ @notifications_bus = notifications_bus
22
+ @namespace = namespace
23
+ @mapped_events = Concurrent::Map.new
24
+ end
25
+
26
+ # Passes the instrumentation block (if any) into the notifications bus
27
+ #
28
+ # @param event_id [String, Symbol] event id
29
+ # @param payload [Hash]
30
+ # @param block [Proc] block we want to instrument (if any)
31
+ def instrument(event_id, payload = EMPTY_HASH, &block)
32
+ full_event_name = @mapped_events[event_id] ||= [event_id, @namespace].compact.join('.')
33
+
34
+ @notifications_bus.instrument(full_event_name, payload, &block)
35
+ end
36
+
37
+ # Allows us to subscribe to the notification bus
38
+ #
39
+ # @param args [Array] any arguments that the notification bus subscription layer accepts
40
+ # @param block [Proc] optional block for subscription
41
+ def subscribe(*args, &block)
42
+ @notifications_bus.subscribe(*args, &block)
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ module Monitoring
6
+ # A simple notifications layer for Karafka ecosystem that aims to provide API compatible
7
+ # with both `ActiveSupport::Notifications` and `dry-monitor`.
8
+ #
9
+ # We do not use any of them by default as our use-case is fairly simple and we do not want
10
+ # to have too many external dependencies.
11
+ class Notifications
12
+ attr_reader :name
13
+
14
+ # Raised when someone wants to publish event that was not registered
15
+ EventNotRegistered = Class.new(StandardError)
16
+
17
+ # Empty hash for internal referencing
18
+ EMPTY_HASH = {}.freeze
19
+
20
+ private_constant :EMPTY_HASH
21
+
22
+ def initialize
23
+ @listeners = Concurrent::Map.new { |k, v| k[v] = Concurrent::Array.new }
24
+ # This allows us to optimize the method calling lookups
25
+ @events_methods_map = Concurrent::Map.new
26
+ end
27
+
28
+ # Registers a new event on which we can publish
29
+ #
30
+ # @param event_id [String, Symbol] event id
31
+ def register_event(event_id)
32
+ @listeners[event_id]
33
+ @events_methods_map[event_id] = :"on_#{event_id.to_s.tr('.', '_')}"
34
+ end
35
+
36
+ # Clears all the subscribed listeners
37
+ def clear
38
+ @listeners.each_value(&:clear)
39
+ end
40
+
41
+ # Allows for subscription to an event
42
+ # There are two ways you can subscribe: via block or via listener.
43
+ #
44
+ # @param event_id_or_listener [Object] event id when we want to subscribe to a particular
45
+ # event with a block or listener if we want to subscribe with general listener
46
+ # @param block [Proc] block of code if we want to subscribe with it
47
+ #
48
+ # @example Subscribe using listener
49
+ # subscribe(MyListener.new)
50
+ #
51
+ # @example Subscribe via block
52
+ # subscribe do |event|
53
+ # puts event
54
+ # end
55
+ def subscribe(event_id_or_listener, &block)
56
+ if block
57
+ event_id = event_id_or_listener
58
+
59
+ raise EventNotRegistered, event_id unless @listeners.key?(event_id)
60
+
61
+ @listeners[event_id] << block
62
+ else
63
+ listener = event_id_or_listener
64
+
65
+ @listeners.each_key do |reg_event_id|
66
+ next unless listener.respond_to?(@events_methods_map[reg_event_id])
67
+
68
+ @listeners[reg_event_id] << listener
69
+ end
70
+ end
71
+ end
72
+
73
+ # Allows for code instrumentation
74
+ # Runs the provided code and sends the instrumentation details to all registered listeners
75
+ #
76
+ # @param event_id [String, Symbol] id of the event
77
+ # @param payload [Hash] payload for the instrumentation
78
+ # @param block [Proc] instrumented code
79
+ # @return [Object] whatever the provided block (if any) returns
80
+ #
81
+ # @example Instrument some code
82
+ # instrument('sleeping') do
83
+ # sleep(1)
84
+ # end
85
+ def instrument(event_id, payload = EMPTY_HASH, &block)
86
+ result, time = measure_time_taken(&block) if block_given?
87
+
88
+ event = Event.new(
89
+ event_id,
90
+ time ? payload.merge(time: time) : payload
91
+ )
92
+
93
+ @listeners[event_id].each do |listener|
94
+ if listener.is_a?(Proc)
95
+ listener.call(event)
96
+ else
97
+ listener.send(@events_methods_map[event_id], event)
98
+ end
99
+ end
100
+
101
+ result
102
+ end
103
+
104
+ private
105
+
106
+ # Measures time taken to execute a given block and returns it together with the result of
107
+ # the block execution
108
+ def measure_time_taken
109
+ start = current_time
110
+ result = yield
111
+ [result, current_time - start]
112
+ end
113
+
114
+ # @return [Integer] current monotonic time
115
+ def current_time
116
+ ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
117
+ end
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Main module namespace
4
+ module Karafka
5
+ module Core
6
+ # Monitoring for Karafka and WaterDrop
7
+ # It allows us to have a layer that can work with `dry-monitor` as well as
8
+ # `ActiveSupport::Notifications` or standalone depending on the case. Thanks to that we do not
9
+ # have to rely on third party tools that could break.
10
+ module Monitoring
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ module Core
5
+ # Current Karafka::Core version
6
+ # We follow the versioning schema of given Karafka version
7
+ VERSION = '2.0.0'
8
+ end
9
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Karafka
4
+ # Namespace for small support modules used throughout the Karafka ecosystem
5
+ module Core
6
+ class << self
7
+ # @return [String] root path of this gem
8
+ def gem_root
9
+ Pathname.new(File.expand_path('../..', __dir__))
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ %w[
4
+ yaml
5
+ concurrent/map
6
+ concurrent/hash
7
+ concurrent/array
8
+ karafka/core
9
+ karafka/core/version
10
+ karafka/core/monitoring
11
+ karafka/core/monitoring/event
12
+ karafka/core/monitoring/monitor
13
+ karafka/core/monitoring/notifications
14
+ karafka/core/configurable
15
+ karafka/core/configurable/leaf
16
+ karafka/core/configurable/node
17
+ karafka/core/contractable/contract
18
+ karafka/core/contractable/result
19
+ karafka/core/contractable/rule
20
+ ].each { |dependency| require dependency }
21
+
22
+ # Karafka framework main namespace
23
+ module Karafka
24
+ end
data/log/.gitkeep ADDED
File without changes
data.tar.gz.sig ADDED
@@ -0,0 +1 @@
1
+ �nnDV��c�"TP��Wo�5���"��T3��ѕ��������U���ye��)�Pt�