racecar 2.0.0.beta1 → 2.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1def66ffdef1ce2d2c100dea3b74a3d69fc5c17ef0c57071ef25cd7cbe4016ad
4
- data.tar.gz: 8198710b789bfa1d49291b020e652e2ab5a7a80e17ccb4ef6a653b3416702342
3
+ metadata.gz: 40511d6f40dafe30b33700935c54aea628eecdeae41fa4c1f25c1af26a9f398c
4
+ data.tar.gz: 41d62e586808c359db8761b872fe8c2ddbfd116b10e703c347b4b4452bdaea57
5
5
  SHA512:
6
- metadata.gz: f4329d9e10df97ce40c90badf31b65db2379a568b96f90e311e407ce866bbb953e5fccb0a55bf16c006424fe7ecf1e6a6901e513060795f9fabfa7e5b7d36a55
7
- data.tar.gz: b04998e5e3786d1c7ab6d9ba9364643709e47f9ba9f753df59edfa3e967e312755b429c47198e10950ee899e972d8330d9aa414849199501de1d41657da45e97
6
+ metadata.gz: 92479e9d06a5c724041fc1b6556a329eadca406f909c767e9aa1f3f09f42470b9e67d2444ec9aa019459ed325d516d59f84772b49cf26fceded1c157e5937edf
7
+ data.tar.gz: e6f60c739ad0cd1db7d833fd32dd7df2537a26830b8d87ac19382aa2bc2d98f1289fe07544159d35d6cdd3dd586a80397068256a36ad5228a3bdd4b76ca143c8
@@ -1,5 +1,6 @@
1
1
  require "logger"
2
2
 
3
+ require "racecar/instrumenter"
3
4
  require "racecar/null_instrumenter"
4
5
  require "racecar/consumer"
5
6
  require "racecar/consumer_set"
@@ -35,13 +36,15 @@ module Racecar
35
36
  end
36
37
 
37
38
  def self.instrumenter
38
- require "active_support/notifications"
39
-
40
- ActiveSupport::Notifications
41
- rescue LoadError
42
- logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
43
-
44
- NullInstrumenter
39
+ @instrumenter ||= begin
40
+ default_payload = { client_id: config.client_id, group_id: config.group_id }
41
+
42
+ Instrumenter.new(default_payload).tap do |instrumenter|
43
+ if instrumenter.backend == NullInstrumenter
44
+ logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
45
+ end
46
+ end
47
+ end
45
48
  end
46
49
 
47
50
  def self.run(processor)
@@ -144,13 +144,14 @@ module Racecar
144
144
  end
145
145
 
146
146
  def configure_datadog
147
- require "kafka/datadog"
147
+ require_relative './datadog'
148
148
 
149
- datadog = Kafka::Datadog
150
- datadog.host = config.datadog_host unless config.datadog_host.nil?
151
- datadog.port = config.datadog_port unless config.datadog_port.nil?
152
- datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
153
- datadog.tags = config.datadog_tags unless config.datadog_tags.nil?
149
+ Datadog.configure do |datadog|
150
+ datadog.host = config.datadog_host unless config.datadog_host.nil?
151
+ datadog.port = config.datadog_port unless config.datadog_port.nil?
152
+ datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
153
+ datadog.tags = config.datadog_tags unless config.datadog_tags.nil?
154
+ end
154
155
  end
155
156
  end
156
157
  end
@@ -41,7 +41,13 @@ module Racecar
41
41
  # Delivers messages that got produced.
42
42
  def deliver!
43
43
  @delivery_handles ||= []
44
- @delivery_handles.each(&:wait)
44
+ if @delivery_handles.any?
45
+ instrumentation_payload = { delivered_message_count: @delivery_handles.size }
46
+
47
+ @instrumenter.instrument('deliver_messages', instrumentation_payload) do
48
+ @delivery_handles.each(&:wait)
49
+ end
50
+ end
45
51
  @delivery_handles.clear
46
52
  end
47
53
 
@@ -50,14 +56,18 @@ module Racecar
50
56
  # https://github.com/appsignal/rdkafka-ruby#producing-messages
51
57
  def produce(payload, topic:, key:, headers: nil, create_time: nil)
52
58
  @delivery_handles ||= []
53
-
54
- extra_info = {
55
- value: payload,
56
- key: key,
57
- topic: topic,
58
- create_time: Time.now,
59
+ message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
60
+ instrumentation_payload = {
61
+ value: payload,
62
+ headers: headers,
63
+ key: key,
64
+ topic: topic,
65
+ message_size: message_size,
66
+ create_time: Time.now,
67
+ buffer_size: @delivery_handles.size
59
68
  }
60
- @instrumenter.instrument("produce_message.racecar", extra_info) do
69
+
70
+ @instrumenter.instrument("produce_message", instrumentation_payload) do
61
71
  @delivery_handles << @producer.produce(topic: topic, payload: payload, key: key, timestamp: create_time, headers: headers)
62
72
  end
63
73
  end
@@ -2,8 +2,9 @@ module Racecar
2
2
  class ConsumerSet
3
3
  MAX_POLL_TRIES = 10
4
4
 
5
- def initialize(config, logger)
5
+ def initialize(config, logger, instrumenter = NullInstrumenter)
6
6
  @config, @logger = config, logger
7
+ @instrumenter = instrumenter
7
8
  raise ArgumentError, "Subscriptions must not be empty when subscribing" if @config.subscriptions.empty?
8
9
 
9
10
  @consumers = []
@@ -71,7 +72,9 @@ module Racecar
71
72
  def current
72
73
  @consumers[@consumer_id_iterator.peek] ||= begin
73
74
  consumer = Rdkafka::Config.new(rdkafka_config(current_subscription)).consumer
74
- consumer.subscribe current_subscription.topic
75
+ @instrumenter.instrument('join_group') do
76
+ consumer.subscribe current_subscription.topic
77
+ end
75
78
  consumer
76
79
  end
77
80
  end
@@ -0,0 +1,245 @@
1
+ begin
2
+ require "datadog/statsd"
3
+ rescue LoadError
4
+ $stderr.puts "In order to report Kafka client metrics to Datadog you need to install the `dogstatsd-ruby` gem."
5
+ raise
6
+ end
7
+
8
+ require "active_support/subscriber"
9
+
10
+ module Racecar
11
+ module Datadog
12
+ STATSD_NAMESPACE = "racecar"
13
+
14
+ class << self
15
+ def configure
16
+ yield self
17
+ end
18
+
19
+ def statsd
20
+ @statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
21
+ end
22
+
23
+ def statsd=(statsd)
24
+ clear
25
+ @statsd = statsd
26
+ end
27
+
28
+ def host
29
+ @host ||= default_host
30
+ end
31
+
32
+ def host=(host)
33
+ @host = host
34
+ clear
35
+ end
36
+
37
+ def port
38
+ @port ||= default_port
39
+ end
40
+
41
+ def port=(port)
42
+ @port = port
43
+ clear
44
+ end
45
+
46
+ def namespace
47
+ @namespace ||= STATSD_NAMESPACE
48
+ end
49
+
50
+ def namespace=(namespace)
51
+ @namespace = namespace
52
+ clear
53
+ end
54
+
55
+ def tags
56
+ @tags ||= []
57
+ end
58
+
59
+ def tags=(tags)
60
+ @tags = tags
61
+ clear
62
+ end
63
+
64
+ private
65
+
66
+ def default_host
67
+ if ::Datadog::Statsd.const_defined?(:Connection)
68
+ ::Datadog::Statsd::Connection::DEFAULT_HOST
69
+ else
70
+ ::Datadog::Statsd::DEFAULT_HOST
71
+ end
72
+ end
73
+
74
+ def default_port
75
+ if ::Datadog::Statsd.const_defined?(:Connection)
76
+ ::Datadog::Statsd::Connection::DEFAULT_PORT
77
+ else
78
+ ::Datadog::Statsd::DEFAULT_PORT
79
+ end
80
+ end
81
+
82
+ def clear
83
+ @statsd && @statsd.close
84
+ @statsd = nil
85
+ end
86
+ end
87
+
88
+ class StatsdSubscriber < ActiveSupport::Subscriber
89
+ private
90
+
91
+ %w[increment histogram count timing gauge].each do |type|
92
+ define_method(type) do |*args|
93
+ emit(type, *args)
94
+ end
95
+ end
96
+
97
+ def emit(type, *args, tags: {})
98
+ tags = tags.map {|k, v| "#{k}:#{v}" }.to_a
99
+
100
+ Racecar::Datadog.statsd.send(type, *args, tags: tags)
101
+ end
102
+ end
103
+
104
+ class ConsumerSubscriber < StatsdSubscriber
105
+ def process_message(event)
106
+ offset = event.payload.fetch(:offset)
107
+ create_time = event.payload.fetch(:create_time)
108
+ time_lag = create_time && ((Time.now - create_time) * 1000).to_i
109
+ tags = default_tags(event)
110
+
111
+ if event.payload.key?(:exception)
112
+ increment("consumer.process_message.errors", tags: tags)
113
+ else
114
+ timing("consumer.process_message.latency", event.duration, tags: tags)
115
+ increment("consumer.messages", tags: tags)
116
+ end
117
+
118
+ gauge("consumer.offset", offset, tags: tags)
119
+
120
+ # Not all messages have timestamps.
121
+ if time_lag
122
+ gauge("consumer.time_lag", time_lag, tags: tags)
123
+ end
124
+ end
125
+
126
+ def process_batch(event)
127
+ offset = event.payload.fetch(:last_offset)
128
+ messages = event.payload.fetch(:message_count)
129
+ tags = default_tags(event)
130
+
131
+ if event.payload.key?(:exception)
132
+ increment("consumer.process_batch.errors", tags: tags)
133
+ else
134
+ timing("consumer.process_batch.latency", event.duration, tags: tags)
135
+ count("consumer.messages", messages, tags: tags)
136
+ end
137
+
138
+ gauge("consumer.offset", offset, tags: tags)
139
+ end
140
+
141
+ def join_group(event)
142
+ tags = {
143
+ client: event.payload.fetch(:client_id),
144
+ group_id: event.payload.fetch(:group_id),
145
+ }
146
+
147
+ timing("consumer.join_group", event.duration, tags: tags)
148
+
149
+ if event.payload.key?(:exception)
150
+ increment("consumer.join_group.errors", tags: tags)
151
+ end
152
+ end
153
+
154
+ def leave_group(event)
155
+ tags = {
156
+ client: event.payload.fetch(:client_id),
157
+ group_id: event.payload.fetch(:group_id),
158
+ }
159
+
160
+ timing("consumer.leave_group", event.duration, tags: tags)
161
+
162
+ if event.payload.key?(:exception)
163
+ increment("consumer.leave_group.errors", tags: tags)
164
+ end
165
+ end
166
+
167
+ def main_loop(event)
168
+ tags = {
169
+ client: event.payload.fetch(:client_id),
170
+ group_id: event.payload.fetch(:group_id),
171
+ }
172
+
173
+ histogram("consumer.loop.duration", event.duration, tags: tags)
174
+ end
175
+
176
+ def pause_status(event)
177
+ duration = event.payload.fetch(:duration)
178
+
179
+ gauge("consumer.pause.duration", duration, tags: default_tags(event))
180
+ end
181
+
182
+ private
183
+
184
+ def default_tags(event)
185
+ {
186
+ client: event.payload.fetch(:client_id),
187
+ group_id: event.payload.fetch(:group_id),
188
+ topic: event.payload.fetch(:topic),
189
+ partition: event.payload.fetch(:partition),
190
+ }
191
+ end
192
+
193
+ attach_to "racecar"
194
+ end
195
+
196
+ class ProducerSubscriber < StatsdSubscriber
197
+ def produce_message(event)
198
+ client = event.payload.fetch(:client_id)
199
+ topic = event.payload.fetch(:topic)
200
+ message_size = event.payload.fetch(:message_size)
201
+ buffer_size = event.payload.fetch(:buffer_size)
202
+
203
+ tags = {
204
+ client: client,
205
+ topic: topic,
206
+ }
207
+
208
+ # This gets us the write rate.
209
+ increment("producer.produce.messages", tags: tags.merge(topic: topic))
210
+
211
+ # Information about typical/average/95p message size.
212
+ histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
213
+
214
+ # Aggregate message size.
215
+ count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
216
+
217
+ # This gets us the avg/max buffer size per producer.
218
+ histogram("producer.buffer.size", buffer_size, tags: tags)
219
+ end
220
+
221
+ def deliver_messages(event)
222
+ client = event.payload.fetch(:client_id)
223
+ message_count = event.payload.fetch(:delivered_message_count)
224
+
225
+ tags = {
226
+ client: client,
227
+ }
228
+
229
+ timing("producer.deliver.latency", event.duration, tags: tags)
230
+
231
+ # Messages delivered to Kafka:
232
+ count("producer.deliver.messages", message_count, tags: tags)
233
+ end
234
+
235
+ def acknowledged_message(event)
236
+ tags = { client: event.payload.fetch(:client_id) }
237
+
238
+ # Number of messages ACK'd for the topic.
239
+ increment("producer.ack.messages", tags: tags)
240
+ end
241
+
242
+ attach_to "producer.racecar"
243
+ end
244
+ end
245
+ end
@@ -0,0 +1,24 @@
1
+ module Racecar
2
+ ##
3
+ # Common API for instrumentation to standardize
4
+ # namespace and default payload
5
+ #
6
+ class Instrumenter
7
+ NAMESPACE = "racecar"
8
+ attr_reader :backend
9
+
10
+ def initialize(default_payload = {})
11
+ @default_payload = default_payload
12
+
13
+ @backend = if defined?(ActiveSupport::Notifications)
14
+ ActiveSupport::Notifications
15
+ else
16
+ NullInstrumenter
17
+ end
18
+ end
19
+
20
+ def instrument(event_name, payload = {}, &block)
21
+ @backend.instrument("#{event_name}.#{NAMESPACE}", @default_payload.merge(payload), &block)
22
+ end
23
+ end
24
+ end
@@ -53,13 +53,16 @@ module Racecar
53
53
  instrumenter: @instrumenter,
54
54
  )
55
55
 
56
- instrument_payload = { consumer_class: processor.class.to_s, consumer_set: consumer }
56
+ instrumentation_payload = {
57
+ consumer_class: processor.class.to_s,
58
+ consumer_set: consumer
59
+ }
57
60
 
58
61
  # Main loop
59
62
  loop do
60
63
  break if @stop_requested
61
64
  resume_paused_partitions
62
- @instrumenter.instrument("main_loop.racecar", instrument_payload) do
65
+ @instrumenter.instrument("main_loop", instrumentation_payload) do
63
66
  case process_method
64
67
  when :batch then
65
68
  msg_per_part = consumer.batch_poll(config.max_wait_time).group_by(&:partition)
@@ -77,7 +80,9 @@ module Racecar
77
80
  processor.deliver!
78
81
  processor.teardown
79
82
  consumer.commit
80
- consumer.close
83
+ @instrumenter.instrument('leave_group') do
84
+ consumer.close
85
+ end
81
86
  end
82
87
 
83
88
  def stop
@@ -106,7 +111,7 @@ module Racecar
106
111
  # a value within librdkafka and is asynchronously written to proper
107
112
  # storage through auto commits.
108
113
  config.consumer << "enable.auto.offset.store=false"
109
- ConsumerSet.new(config, logger)
114
+ ConsumerSet.new(config, logger, @instrumenter)
110
115
  end
111
116
  end
112
117
 
@@ -130,8 +135,11 @@ module Racecar
130
135
 
131
136
  def delivery_callback
132
137
  ->(delivery_report) do
133
- data = {offset: delivery_report.offset, partition: delivery_report.partition}
134
- @instrumenter.instrument("acknowledged_message.racecar", data)
138
+ payload = {
139
+ offset: delivery_report.offset,
140
+ partition: delivery_report.partition
141
+ }
142
+ @instrumenter.instrument("acknowledged_message", payload)
135
143
  end
136
144
  end
137
145
 
@@ -146,14 +154,19 @@ module Racecar
146
154
  end
147
155
 
148
156
  def process(message)
149
- payload = {
157
+ instrumentation_payload = {
150
158
  consumer_class: processor.class.to_s,
151
- topic: message.topic,
152
- partition: message.partition,
153
- offset: message.offset,
159
+ topic: message.topic,
160
+ partition: message.partition,
161
+ offset: message.offset,
162
+ create_time: message.timestamp,
163
+ key: message.key,
164
+ value: message.payload,
165
+ headers: message.headers
154
166
  }
155
167
 
156
- @instrumenter.instrument("process_message.racecar", payload) do
168
+ @instrumenter.instrument("start_process_message", instrumentation_payload)
169
+ @instrumenter.instrument("process_message", instrumentation_payload) do
157
170
  with_pause(message.topic, message.partition, message.offset..message.offset) do
158
171
  processor.process(Racecar::Message.new(message))
159
172
  processor.deliver!
@@ -163,16 +176,18 @@ module Racecar
163
176
  end
164
177
 
165
178
  def process_batch(messages)
166
- payload = {
179
+ first, last = messages.first, messages.last
180
+ instrumentation_payload = {
167
181
  consumer_class: processor.class.to_s,
168
- topic: messages.first.topic,
169
- partition: messages.first.partition,
170
- first_offset: messages.first.offset,
171
- message_count: messages.size,
182
+ topic: first.topic,
183
+ partition: first.partition,
184
+ first_offset: first.offset,
185
+ last_offset: last.offset,
186
+ message_count: messages.size
172
187
  }
173
188
 
174
- @instrumenter.instrument("process_batch.racecar", payload) do
175
- first, last = messages.first, messages.last
189
+ @instrumenter.instrument("start_process_batch", instrumentation_payload)
190
+ @instrumenter.instrument("process_batch", instrumentation_payload) do
176
191
  with_pause(first.topic, first.partition, first.offset..last.offset) do
177
192
  processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
178
193
  processor.deliver!
@@ -204,11 +219,13 @@ module Racecar
204
219
 
205
220
  pauses.each do |topic, partitions|
206
221
  partitions.each do |partition, pause|
207
- @instrumenter.instrument("pause_status.racecar", {
208
- topic: topic,
209
- partition: partition,
210
- duration: pause.pause_duration,
211
- })
222
+ instrumentation_payload = {
223
+ topic: topic,
224
+ partition: partition,
225
+ duration: pause.pause_duration,
226
+ consumer_class: processor.class.to_s,
227
+ }
228
+ @instrumenter.instrument("pause_status", instrumentation_payload)
212
229
 
213
230
  if pause.paused? && pause.expired?
214
231
  logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
@@ -1,3 +1,3 @@
1
1
  module Racecar
2
- VERSION = "2.0.0.beta1"
2
+ VERSION = "2.0.0.beta2"
3
3
  end
@@ -27,4 +27,6 @@ Gem::Specification.new do |spec|
27
27
  spec.add_development_dependency "rake", "> 10.0"
28
28
  spec.add_development_dependency "rspec", "~> 3.0"
29
29
  spec.add_development_dependency "timecop"
30
+ spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
31
+ spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
30
32
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: racecar
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.beta1
4
+ version: 2.0.0.beta2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2020-01-08 00:00:00.000000000 Z
12
+ date: 2020-02-05 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: king_konf
@@ -101,6 +101,46 @@ dependencies:
101
101
  - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: '0'
104
+ - !ruby/object:Gem::Dependency
105
+ name: dogstatsd-ruby
106
+ requirement: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: 3.0.0
111
+ - - "<"
112
+ - !ruby/object:Gem::Version
113
+ version: 5.0.0
114
+ type: :development
115
+ prerelease: false
116
+ version_requirements: !ruby/object:Gem::Requirement
117
+ requirements:
118
+ - - ">="
119
+ - !ruby/object:Gem::Version
120
+ version: 3.0.0
121
+ - - "<"
122
+ - !ruby/object:Gem::Version
123
+ version: 5.0.0
124
+ - !ruby/object:Gem::Dependency
125
+ name: activesupport
126
+ requirement: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - ">="
129
+ - !ruby/object:Gem::Version
130
+ version: '4.0'
131
+ - - "<"
132
+ - !ruby/object:Gem::Version
133
+ version: '6.1'
134
+ type: :development
135
+ prerelease: false
136
+ version_requirements: !ruby/object:Gem::Requirement
137
+ requirements:
138
+ - - ">="
139
+ - !ruby/object:Gem::Version
140
+ version: '4.0'
141
+ - - "<"
142
+ - !ruby/object:Gem::Version
143
+ version: '6.1'
104
144
  description:
105
145
  email:
106
146
  - dschierbeck@zendesk.com
@@ -139,6 +179,8 @@ files:
139
179
  - lib/racecar/consumer_set.rb
140
180
  - lib/racecar/ctl.rb
141
181
  - lib/racecar/daemon.rb
182
+ - lib/racecar/datadog.rb
183
+ - lib/racecar/instrumenter.rb
142
184
  - lib/racecar/message.rb
143
185
  - lib/racecar/null_instrumenter.rb
144
186
  - lib/racecar/pause.rb