deimos-ruby 1.3.0.pre.beta5 → 1.4.0.pre.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -9
- data/Gemfile.lock +2 -88
- data/README.md +18 -135
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +0 -1
- data/docs/CONFIGURATION.md +200 -0
- data/lib/deimos/avro_data_coder.rb +4 -4
- data/lib/deimos/backends/db.rb +0 -5
- data/lib/deimos/base_consumer.rb +2 -2
- data/lib/deimos/config/configurable.rb +258 -0
- data/lib/deimos/config/configuration.rb +354 -0
- data/lib/deimos/config/phobos_config.rb +134 -0
- data/lib/deimos/kafka_message.rb +6 -0
- data/lib/deimos/producer.rb +5 -5
- data/lib/deimos/test_helpers.rb +2 -10
- data/lib/deimos/utils/db_producer.rb +0 -5
- data/lib/deimos/utils/lag_reporter.rb +4 -4
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +2 -86
- data/lib/tasks/deimos.rake +1 -1
- data/spec/backends/db_spec.rb +0 -5
- data/spec/batch_consumer_spec.rb +1 -1
- data/spec/config/configurable_spec.rb +121 -0
- data/spec/config/configuration_spec.rb +229 -0
- data/spec/consumer_spec.rb +5 -5
- data/spec/deimos_spec.rb +14 -35
- data/spec/kafka_source_spec.rb +1 -1
- data/spec/producer_spec.rb +5 -5
- data/spec/spec_helper.rb +8 -20
- data/spec/updateable_schema_store_spec.rb +1 -1
- data/spec/utils/db_producer_spec.rb +0 -10
- data/spec/utils/lag_reporter_spec.rb +2 -2
- metadata +10 -19
- data/lib/deimos/configuration.rb +0 -124
- data/spec/rake_spec.rb +0 -19
data/lib/deimos/configuration.rb
DELETED
@@ -1,124 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Deimos
|
4
|
-
# Class to hold configuration.
|
5
|
-
class Configuration
|
6
|
-
# @return [Logger]
|
7
|
-
attr_accessor :logger
|
8
|
-
attr_accessor :phobos_logger
|
9
|
-
attr_accessor :kafka_logger
|
10
|
-
|
11
|
-
# By default, consumer errors will be consumed and logged to
|
12
|
-
# the metrics provider.
|
13
|
-
# Set this to true to force the error to be raised.
|
14
|
-
# @return [Boolean]
|
15
|
-
attr_accessor :reraise_consumer_errors
|
16
|
-
|
17
|
-
# @return [String]
|
18
|
-
attr_accessor :schema_registry_url
|
19
|
-
|
20
|
-
# @return [String]
|
21
|
-
attr_accessor :seed_broker
|
22
|
-
|
23
|
-
# Local path to schemas.
|
24
|
-
# @return [String]
|
25
|
-
attr_accessor :schema_path
|
26
|
-
|
27
|
-
# Default namespace for all producers. Can remain nil. Individual
|
28
|
-
# producers can override.
|
29
|
-
# @return [String]
|
30
|
-
attr_accessor :producer_schema_namespace
|
31
|
-
|
32
|
-
# Add a prefix to all topic names. This can be useful if you're using
|
33
|
-
# the same Kafka broker for different environments that are producing
|
34
|
-
# the same topics.
|
35
|
-
# @return [String]
|
36
|
-
attr_accessor :producer_topic_prefix
|
37
|
-
|
38
|
-
# Disable all actual message producing. Useful when doing things like
|
39
|
-
# mass imports or data space management when events don't need to be
|
40
|
-
# fired.
|
41
|
-
# @return [Boolean]
|
42
|
-
attr_accessor :disable_producers
|
43
|
-
|
44
|
-
# File path to the Phobos configuration file, relative to the application root.
|
45
|
-
# @return [String]
|
46
|
-
attr_accessor :phobos_config_file
|
47
|
-
|
48
|
-
# @return [Boolean]
|
49
|
-
attr_accessor :ssl_enabled
|
50
|
-
|
51
|
-
# @return [String]
|
52
|
-
attr_accessor :ssl_ca_cert
|
53
|
-
|
54
|
-
# @return [String]
|
55
|
-
attr_accessor :ssl_client_cert
|
56
|
-
|
57
|
-
# @return [String]
|
58
|
-
attr_accessor :ssl_client_cert_key
|
59
|
-
|
60
|
-
# Currently can be set to :db, :kafka, or :async_kafka. If using Kafka
|
61
|
-
# directly, set to async in your user-facing app, and sync in your
|
62
|
-
# consumers or delayed workers.
|
63
|
-
# @return [Symbol]
|
64
|
-
attr_accessor :publish_backend
|
65
|
-
|
66
|
-
# @return [Boolean]
|
67
|
-
attr_accessor :report_lag
|
68
|
-
|
69
|
-
# @return [Metrics::Provider]
|
70
|
-
attr_accessor :metrics
|
71
|
-
|
72
|
-
# @return [Tracing::Provider]
|
73
|
-
attr_accessor :tracer
|
74
|
-
|
75
|
-
# @return [Deimos::DbProducerConfiguration]
|
76
|
-
attr_accessor :db_producer
|
77
|
-
|
78
|
-
# For internal purposes only
|
79
|
-
# @return [Block]
|
80
|
-
attr_accessor :fatal_error_block
|
81
|
-
|
82
|
-
# :nodoc:
|
83
|
-
def initialize
|
84
|
-
@phobos_config_file = 'config/phobos.yml'
|
85
|
-
@publish_backend = :kafka_async
|
86
|
-
@db_producer = DbProducerConfiguration.new
|
87
|
-
fatal_error { false }
|
88
|
-
end
|
89
|
-
|
90
|
-
# Block taking an exception, payload and metadata and returning
|
91
|
-
# true if this should be considered a fatal error and false otherwise.
|
92
|
-
# Not needed if reraise_consumer_errors is set to true.
|
93
|
-
def fatal_error(&block)
|
94
|
-
@fatal_error_block = block
|
95
|
-
end
|
96
|
-
|
97
|
-
# @param other_config [Configuration]
|
98
|
-
# @return [Boolean]
|
99
|
-
def phobos_config_changed?(other_config)
|
100
|
-
phobos_keys = %w(seed_broker phobos_config_file ssl_ca_cert ssl_client_cert ssl_client_cert_key)
|
101
|
-
return true if phobos_keys.any? { |key| self.send(key) != other_config.send(key) }
|
102
|
-
|
103
|
-
other_config.logger != self.logger
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
# Sub-class for DB producer configs.
|
108
|
-
class DbProducerConfiguration
|
109
|
-
# @return [Logger]
|
110
|
-
attr_accessor :logger
|
111
|
-
# @return [Symbol|Array<String>] A list of topics to log all messages, or
|
112
|
-
# :all to log all topics.
|
113
|
-
attr_accessor :log_topics
|
114
|
-
# @return [Symbol|Array<String>] A list of topics to compact messages for
|
115
|
-
# before sending, or :all to compact all keyed messages.
|
116
|
-
attr_accessor :compact_topics
|
117
|
-
|
118
|
-
# :nodoc:
|
119
|
-
def initialize
|
120
|
-
@log_topics = []
|
121
|
-
@compact_topics = []
|
122
|
-
end
|
123
|
-
end
|
124
|
-
end
|
data/spec/rake_spec.rb
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'rake'
|
4
|
-
require 'rails'
|
5
|
-
Rails.logger = Logger.new(STDOUT)
|
6
|
-
load("#{__dir__}/../lib/tasks/deimos.rake")
|
7
|
-
|
8
|
-
if Rake.application.lookup(:environment).nil?
|
9
|
-
Rake::Task.define_task(:environment)
|
10
|
-
end
|
11
|
-
|
12
|
-
describe 'Rakefile' do # rubocop:disable RSpec/DescribeClass
|
13
|
-
it 'should start listeners' do
|
14
|
-
runner = instance_double(Phobos::CLI::Runner)
|
15
|
-
expect(Phobos::CLI::Runner).to receive(:new).and_return(runner)
|
16
|
-
expect(runner).to receive(:run!)
|
17
|
-
Rake::Task['deimos:start'].invoke
|
18
|
-
end
|
19
|
-
end
|