waterdrop 1.3.1 → 2.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.diffend.yml +3 -0
- data/.github/workflows/ci.yml +53 -0
- data/.gitignore +4 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +27 -4
- data/Gemfile +9 -0
- data/Gemfile.lock +54 -37
- data/LICENSE +165 -0
- data/README.md +192 -53
- data/certs/mensfeld.pem +21 -21
- data/config/errors.yml +3 -16
- data/docker-compose.yml +17 -0
- data/lib/water_drop.rb +5 -23
- data/lib/water_drop/config.rb +41 -136
- data/lib/water_drop/contracts.rb +0 -2
- data/lib/water_drop/contracts/config.rb +8 -121
- data/lib/water_drop/contracts/message.rb +41 -0
- data/lib/water_drop/errors.rb +30 -5
- data/lib/water_drop/instrumentation.rb +7 -0
- data/lib/water_drop/instrumentation/monitor.rb +16 -23
- data/lib/water_drop/instrumentation/stdout_listener.rb +113 -32
- data/lib/water_drop/producer.rb +142 -0
- data/lib/water_drop/producer/async.rb +51 -0
- data/lib/water_drop/producer/buffer.rb +113 -0
- data/lib/water_drop/producer/builder.rb +63 -0
- data/lib/water_drop/producer/dummy_client.rb +32 -0
- data/lib/water_drop/producer/statistics_decorator.rb +71 -0
- data/lib/water_drop/producer/status.rb +52 -0
- data/lib/water_drop/producer/sync.rb +65 -0
- data/lib/water_drop/version.rb +1 -1
- data/log/{.keep → .gitkeep} +0 -0
- data/waterdrop.gemspec +4 -4
- metadata +52 -45
- metadata.gz.sig +0 -0
- data/.travis.yml +0 -35
- data/MIT-LICENCE +0 -18
- data/lib/water_drop/async_producer.rb +0 -26
- data/lib/water_drop/base_producer.rb +0 -57
- data/lib/water_drop/config_applier.rb +0 -52
- data/lib/water_drop/contracts/message_options.rb +0 -19
- data/lib/water_drop/sync_producer.rb +0 -24
@@ -1,52 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Engine used to propagate config application to DeliveryBoy with corner case handling
|
5
|
-
module ConfigApplier
|
6
|
-
class << self
|
7
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
8
|
-
# @param settings [Hash] hash with WaterDrop settings
|
9
|
-
def call(delivery_boy_config, settings)
|
10
|
-
# Recursive lambda for mapping config down to delivery boy
|
11
|
-
settings.each do |key, value|
|
12
|
-
call(delivery_boy_config, value) && next if value.is_a?(Hash)
|
13
|
-
|
14
|
-
# If this is a special case that needs manual setup instead of a direct reassignment
|
15
|
-
if respond_to?(key, true)
|
16
|
-
send(key, delivery_boy_config, value)
|
17
|
-
else
|
18
|
-
# If this setting is our internal one, we should not sync it with the delivery boy
|
19
|
-
next unless delivery_boy_config.respond_to?(:"#{key}=")
|
20
|
-
|
21
|
-
delivery_boy_config.public_send(:"#{key}=", value)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
# Extra setup for the compression codec as it behaves differently than other settings
|
29
|
-
# that are ported 1:1 from ruby-kafka
|
30
|
-
# For some crazy reason, delivery boy requires compression codec as a string, when
|
31
|
-
# ruby-kafka as a symbol. We follow ruby-kafka internal design, so we had to mimic
|
32
|
-
# that by assigning a string version that down the road will be symbolized again
|
33
|
-
# by delivery boy
|
34
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
35
|
-
# @param codec_name [Symbol] codec name as a symbol
|
36
|
-
def compression_codec(delivery_boy_config, codec_name)
|
37
|
-
# If there is no compression codec, we don't apply anything
|
38
|
-
return unless codec_name
|
39
|
-
|
40
|
-
delivery_boy_config.compression_codec = codec_name.to_s
|
41
|
-
end
|
42
|
-
|
43
|
-
# We use the "seed_brokers" name and DeliveryBoy uses "brokers" so we pass the values
|
44
|
-
# manually
|
45
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
46
|
-
# @param seed_brokers [Array<String>] kafka seed brokers
|
47
|
-
def seed_brokers(delivery_boy_config, seed_brokers)
|
48
|
-
delivery_boy_config.brokers = seed_brokers
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Contracts
|
5
|
-
# Contract with validation rules for validating that all the message options that
|
6
|
-
# we provide to producer ale valid and usable
|
7
|
-
# @note Does not validate message itself as it is not our concern
|
8
|
-
class MessageOptions < Dry::Validation::Contract
|
9
|
-
params do
|
10
|
-
required(:topic).filled(:str?, format?: TOPIC_REGEXP)
|
11
|
-
optional(:key).maybe(:str?, :filled?)
|
12
|
-
optional(:partition).filled(:int?, gteq?: 0)
|
13
|
-
optional(:partition_key).maybe(:str?, :filled?)
|
14
|
-
optional(:create_time).maybe(:time?)
|
15
|
-
optional(:headers).maybe(:hash?)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# WaterDrop library
|
4
|
-
module WaterDrop
|
5
|
-
# Sync producer for messages
|
6
|
-
class SyncProducer < BaseProducer
|
7
|
-
# Performs message delivery using deliver_async method
|
8
|
-
# @param message [String] message that we want to send to Kafka
|
9
|
-
# @param options [Hash] options (including topic) for producer
|
10
|
-
# @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
|
11
|
-
# somehow invalid and we cannot perform delivery because of that
|
12
|
-
def self.call(message, options)
|
13
|
-
attempts_count ||= 0
|
14
|
-
attempts_count += 1
|
15
|
-
|
16
|
-
validate!(options)
|
17
|
-
return unless WaterDrop.config.deliver
|
18
|
-
|
19
|
-
DeliveryBoy.deliver(message, options)
|
20
|
-
rescue Kafka::Error => e
|
21
|
-
graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|