waterdrop 1.3.1 → 2.0.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.diffend.yml +3 -0
- data/.github/workflows/ci.yml +53 -0
- data/.gitignore +4 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +27 -4
- data/Gemfile +9 -0
- data/Gemfile.lock +54 -37
- data/LICENSE +165 -0
- data/README.md +192 -53
- data/certs/mensfeld.pem +21 -21
- data/config/errors.yml +3 -16
- data/docker-compose.yml +17 -0
- data/lib/water_drop.rb +5 -23
- data/lib/water_drop/config.rb +41 -136
- data/lib/water_drop/contracts.rb +0 -2
- data/lib/water_drop/contracts/config.rb +8 -121
- data/lib/water_drop/contracts/message.rb +41 -0
- data/lib/water_drop/errors.rb +30 -5
- data/lib/water_drop/instrumentation.rb +7 -0
- data/lib/water_drop/instrumentation/monitor.rb +16 -23
- data/lib/water_drop/instrumentation/stdout_listener.rb +113 -32
- data/lib/water_drop/producer.rb +142 -0
- data/lib/water_drop/producer/async.rb +51 -0
- data/lib/water_drop/producer/buffer.rb +113 -0
- data/lib/water_drop/producer/builder.rb +63 -0
- data/lib/water_drop/producer/dummy_client.rb +32 -0
- data/lib/water_drop/producer/statistics_decorator.rb +71 -0
- data/lib/water_drop/producer/status.rb +52 -0
- data/lib/water_drop/producer/sync.rb +65 -0
- data/lib/water_drop/version.rb +1 -1
- data/log/{.keep → .gitkeep} +0 -0
- data/waterdrop.gemspec +4 -4
- metadata +52 -45
- metadata.gz.sig +0 -0
- data/.travis.yml +0 -35
- data/MIT-LICENCE +0 -18
- data/lib/water_drop/async_producer.rb +0 -26
- data/lib/water_drop/base_producer.rb +0 -57
- data/lib/water_drop/config_applier.rb +0 -52
- data/lib/water_drop/contracts/message_options.rb +0 -19
- data/lib/water_drop/sync_producer.rb +0 -24
@@ -1,52 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Engine used to propagate config application to DeliveryBoy with corner case handling
|
5
|
-
module ConfigApplier
|
6
|
-
class << self
|
7
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
8
|
-
# @param settings [Hash] hash with WaterDrop settings
|
9
|
-
def call(delivery_boy_config, settings)
|
10
|
-
# Recursive lambda for mapping config down to delivery boy
|
11
|
-
settings.each do |key, value|
|
12
|
-
call(delivery_boy_config, value) && next if value.is_a?(Hash)
|
13
|
-
|
14
|
-
# If this is a special case that needs manual setup instead of a direct reassignment
|
15
|
-
if respond_to?(key, true)
|
16
|
-
send(key, delivery_boy_config, value)
|
17
|
-
else
|
18
|
-
# If this setting is our internal one, we should not sync it with the delivery boy
|
19
|
-
next unless delivery_boy_config.respond_to?(:"#{key}=")
|
20
|
-
|
21
|
-
delivery_boy_config.public_send(:"#{key}=", value)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
# Extra setup for the compression codec as it behaves differently than other settings
|
29
|
-
# that are ported 1:1 from ruby-kafka
|
30
|
-
# For some crazy reason, delivery boy requires compression codec as a string, when
|
31
|
-
# ruby-kafka as a symbol. We follow ruby-kafka internal design, so we had to mimic
|
32
|
-
# that by assigning a string version that down the road will be symbolized again
|
33
|
-
# by delivery boy
|
34
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
35
|
-
# @param codec_name [Symbol] codec name as a symbol
|
36
|
-
def compression_codec(delivery_boy_config, codec_name)
|
37
|
-
# If there is no compression codec, we don't apply anything
|
38
|
-
return unless codec_name
|
39
|
-
|
40
|
-
delivery_boy_config.compression_codec = codec_name.to_s
|
41
|
-
end
|
42
|
-
|
43
|
-
# We use the "seed_brokers" name and DeliveryBoy uses "brokers" so we pass the values
|
44
|
-
# manually
|
45
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
46
|
-
# @param seed_brokers [Array<String>] kafka seed brokers
|
47
|
-
def seed_brokers(delivery_boy_config, seed_brokers)
|
48
|
-
delivery_boy_config.brokers = seed_brokers
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Contracts
|
5
|
-
# Contract with validation rules for validating that all the message options that
|
6
|
-
# we provide to producer ale valid and usable
|
7
|
-
# @note Does not validate message itself as it is not our concern
|
8
|
-
class MessageOptions < Dry::Validation::Contract
|
9
|
-
params do
|
10
|
-
required(:topic).filled(:str?, format?: TOPIC_REGEXP)
|
11
|
-
optional(:key).maybe(:str?, :filled?)
|
12
|
-
optional(:partition).filled(:int?, gteq?: 0)
|
13
|
-
optional(:partition_key).maybe(:str?, :filled?)
|
14
|
-
optional(:create_time).maybe(:time?)
|
15
|
-
optional(:headers).maybe(:hash?)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# WaterDrop library
|
4
|
-
module WaterDrop
|
5
|
-
# Sync producer for messages
|
6
|
-
class SyncProducer < BaseProducer
|
7
|
-
# Performs message delivery using deliver_async method
|
8
|
-
# @param message [String] message that we want to send to Kafka
|
9
|
-
# @param options [Hash] options (including topic) for producer
|
10
|
-
# @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
|
11
|
-
# somehow invalid and we cannot perform delivery because of that
|
12
|
-
def self.call(message, options)
|
13
|
-
attempts_count ||= 0
|
14
|
-
attempts_count += 1
|
15
|
-
|
16
|
-
validate!(options)
|
17
|
-
return unless WaterDrop.config.deliver
|
18
|
-
|
19
|
-
DeliveryBoy.deliver(message, options)
|
20
|
-
rescue Kafka::Error => e
|
21
|
-
graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|