racecar 2.0.0.beta1 → 2.0.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1def66ffdef1ce2d2c100dea3b74a3d69fc5c17ef0c57071ef25cd7cbe4016ad
-  data.tar.gz: 8198710b789bfa1d49291b020e652e2ab5a7a80e17ccb4ef6a653b3416702342
+  metadata.gz: 40511d6f40dafe30b33700935c54aea628eecdeae41fa4c1f25c1af26a9f398c
+  data.tar.gz: 41d62e586808c359db8761b872fe8c2ddbfd116b10e703c347b4b4452bdaea57
 SHA512:
-  metadata.gz: f4329d9e10df97ce40c90badf31b65db2379a568b96f90e311e407ce866bbb953e5fccb0a55bf16c006424fe7ecf1e6a6901e513060795f9fabfa7e5b7d36a55
-  data.tar.gz: b04998e5e3786d1c7ab6d9ba9364643709e47f9ba9f753df59edfa3e967e312755b429c47198e10950ee899e972d8330d9aa414849199501de1d41657da45e97
+  metadata.gz: 92479e9d06a5c724041fc1b6556a329eadca406f909c767e9aa1f3f09f42470b9e67d2444ec9aa019459ed325d516d59f84772b49cf26fceded1c157e5937edf
+  data.tar.gz: e6f60c739ad0cd1db7d833fd32dd7df2537a26830b8d87ac19382aa2bc2d98f1289fe07544159d35d6cdd3dd586a80397068256a36ad5228a3bdd4b76ca143c8

data/lib/racecar.rb

@@ -1,5 +1,6 @@
 require "logger"
 
+require "racecar/instrumenter"
 require "racecar/null_instrumenter"
 require "racecar/consumer"
 require "racecar/consumer_set"
@@ -35,13 +36,15 @@ module Racecar
   end
 
   def self.instrumenter
-    require "active_support/notifications"
-
-    ActiveSupport::Notifications
-  rescue LoadError
-    logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
-
-    NullInstrumenter
+    @instrumenter ||= begin
+      default_payload = { client_id: config.client_id, group_id: config.group_id }
+
+      Instrumenter.new(default_payload).tap do |instrumenter|
+        if instrumenter.backend == NullInstrumenter
+          logger.warn "ActiveSupport::Notifications not available, instrumentation is disabled"
+        end
+      end
+    end
   end
 
   def self.run(processor)

data/lib/racecar/cli.rb

@@ -144,13 +144,14 @@ module Racecar
     end
 
     def configure_datadog
-      require "kafka/datadog"
+      require_relative './datadog'
 
-      datadog = Kafka::Datadog
-      datadog.host = config.datadog_host unless config.datadog_host.nil?
-      datadog.port = config.datadog_port unless config.datadog_port.nil?
-      datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
-      datadog.tags = config.datadog_tags unless config.datadog_tags.nil?
+      Datadog.configure do |datadog|
+        datadog.host      = config.datadog_host unless config.datadog_host.nil?
+        datadog.port      = config.datadog_port unless config.datadog_port.nil?
+        datadog.namespace = config.datadog_namespace unless config.datadog_namespace.nil?
+        datadog.tags      = config.datadog_tags unless config.datadog_tags.nil?
+      end
     end
   end
 end

data/lib/racecar/consumer.rb

@@ -41,7 +41,13 @@ module Racecar
     # Delivers messages that got produced.
     def deliver!
       @delivery_handles ||= []
-      @delivery_handles.each(&:wait)
+      if @delivery_handles.any?
+        instrumentation_payload = { delivered_message_count: @delivery_handles.size }
+
+        @instrumenter.instrument('deliver_messages', instrumentation_payload) do
+          @delivery_handles.each(&:wait)
+        end
+      end
       @delivery_handles.clear
     end
 
@@ -50,14 +56,18 @@ module Racecar
     # https://github.com/appsignal/rdkafka-ruby#producing-messages
     def produce(payload, topic:, key:, headers: nil, create_time: nil)
       @delivery_handles ||= []
-
-      extra_info = {
-        value:       payload,
-        key:         key,
-        topic:       topic,
-        create_time: Time.now,
+      message_size = payload.respond_to?(:bytesize) ? payload.bytesize : 0
+      instrumentation_payload = {
+        value:        payload,
+        headers:      headers,
+        key:          key,
+        topic:        topic,
+        message_size: message_size,
+        create_time:  Time.now,
+        buffer_size:  @delivery_handles.size
       }
-      @instrumenter.instrument("produce_message.racecar", extra_info) do
+
+      @instrumenter.instrument("produce_message", instrumentation_payload) do
         @delivery_handles << @producer.produce(topic: topic, payload: payload, key: key, timestamp: create_time, headers: headers)
       end
     end

data/lib/racecar/consumer_set.rb

@@ -2,8 +2,9 @@ module Racecar
   class ConsumerSet
     MAX_POLL_TRIES = 10
 
-    def initialize(config, logger)
+    def initialize(config, logger, instrumenter = NullInstrumenter)
       @config, @logger = config, logger
+      @instrumenter = instrumenter
       raise ArgumentError, "Subscriptions must not be empty when subscribing" if @config.subscriptions.empty?
 
       @consumers = []
@@ -71,7 +72,9 @@ module Racecar
     def current
       @consumers[@consumer_id_iterator.peek] ||= begin
         consumer = Rdkafka::Config.new(rdkafka_config(current_subscription)).consumer
-        consumer.subscribe current_subscription.topic
+        @instrumenter.instrument('join_group') do
+          consumer.subscribe current_subscription.topic
+        end
         consumer
       end
     end

data/lib/racecar/datadog.rb

@@ -0,0 +1,245 @@
+begin
+  require "datadog/statsd"
+rescue LoadError
+  $stderr.puts "In order to report Kafka client metrics to Datadog you need to install the `dogstatsd-ruby` gem."
+  raise
+end
+
+require "active_support/subscriber"
+
+module Racecar
+  module Datadog
+    STATSD_NAMESPACE = "racecar"
+
+    class << self
+      def configure
+        yield self
+      end
+
+      def statsd
+        @statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
+      end
+
+      def statsd=(statsd)
+        clear
+        @statsd = statsd
+      end
+
+      def host
+        @host ||= default_host
+      end
+
+      def host=(host)
+        @host = host
+        clear
+      end
+
+      def port
+        @port ||= default_port
+      end
+
+      def port=(port)
+        @port = port
+        clear
+      end
+
+      def namespace
+        @namespace ||= STATSD_NAMESPACE
+      end
+
+      def namespace=(namespace)
+        @namespace = namespace
+        clear
+      end
+
+      def tags
+        @tags ||= []
+      end
+
+      def tags=(tags)
+        @tags = tags
+        clear
+      end
+
+      private
+
+      def default_host
+        if ::Datadog::Statsd.const_defined?(:Connection)
+          ::Datadog::Statsd::Connection::DEFAULT_HOST
+        else
+          ::Datadog::Statsd::DEFAULT_HOST
+        end
+      end
+
+      def default_port
+        if ::Datadog::Statsd.const_defined?(:Connection)
+          ::Datadog::Statsd::Connection::DEFAULT_PORT
+        else
+          ::Datadog::Statsd::DEFAULT_PORT
+        end
+      end
+
+      def clear
+        @statsd && @statsd.close
+        @statsd = nil
+      end
+    end
+
+    class StatsdSubscriber < ActiveSupport::Subscriber
+      private
+
+      %w[increment histogram count timing gauge].each do |type|
+        define_method(type) do |*args|
+          emit(type, *args)
+        end
+      end
+
+      def emit(type, *args, tags: {})
+        tags = tags.map {|k, v| "#{k}:#{v}" }.to_a
+
+        Racecar::Datadog.statsd.send(type, *args, tags: tags)
+      end
+    end
+
+    class ConsumerSubscriber < StatsdSubscriber
+      def process_message(event)
+        offset = event.payload.fetch(:offset)
+        create_time = event.payload.fetch(:create_time)
+        time_lag = create_time && ((Time.now - create_time) * 1000).to_i
+        tags = default_tags(event)
+
+        if event.payload.key?(:exception)
+          increment("consumer.process_message.errors", tags: tags)
+        else
+          timing("consumer.process_message.latency", event.duration, tags: tags)
+          increment("consumer.messages", tags: tags)
+        end
+
+        gauge("consumer.offset", offset, tags: tags)
+
+        # Not all messages have timestamps.
+        if time_lag
+          gauge("consumer.time_lag", time_lag, tags: tags)
+        end
+      end
+
+      def process_batch(event)
+        offset = event.payload.fetch(:last_offset)
+        messages = event.payload.fetch(:message_count)
+        tags = default_tags(event)
+
+        if event.payload.key?(:exception)
+          increment("consumer.process_batch.errors", tags: tags)
+        else
+          timing("consumer.process_batch.latency", event.duration, tags: tags)
+          count("consumer.messages", messages, tags: tags)
+        end
+
+        gauge("consumer.offset", offset, tags: tags)
+      end
+
+      def join_group(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+
+        timing("consumer.join_group", event.duration, tags: tags)
+
+        if event.payload.key?(:exception)
+          increment("consumer.join_group.errors", tags: tags)
+        end
+      end
+
+      def leave_group(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+
+        timing("consumer.leave_group", event.duration, tags: tags)
+
+        if event.payload.key?(:exception)
+          increment("consumer.leave_group.errors", tags: tags)
+        end
+      end
+
+      def main_loop(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+        }
+
+        histogram("consumer.loop.duration", event.duration, tags: tags)
+      end
+
+      def pause_status(event)
+        duration = event.payload.fetch(:duration)
+
+        gauge("consumer.pause.duration", duration, tags: default_tags(event))
+      end
+
+      private
+
+      def default_tags(event)
+        {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition),
+        }
+      end
+
+      attach_to "racecar"
+    end
+
+    class ProducerSubscriber < StatsdSubscriber
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
+        buffer_size = event.payload.fetch(:buffer_size)
+
+        tags = {
+          client: client,
+          topic: topic,
+        }
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # Information about typical/average/95p message size.
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+
+        # This gets us the avg/max buffer size per producer.
+        histogram("producer.buffer.size", buffer_size, tags: tags)
+      end
+
+      def deliver_messages(event)
+        client = event.payload.fetch(:client_id)
+        message_count = event.payload.fetch(:delivered_message_count)
+
+        tags = {
+          client: client,
+        }
+
+        timing("producer.deliver.latency", event.duration, tags: tags)
+
+        # Messages delivered to Kafka:
+        count("producer.deliver.messages", message_count, tags: tags)
+      end
+
+      def acknowledged_message(event)
+        tags = { client: event.payload.fetch(:client_id) }
+
+        # Number of messages ACK'd for the topic.
+        increment("producer.ack.messages", tags: tags)
+      end
+
+      attach_to "producer.racecar"
+    end
+  end
+end

data/lib/racecar/instrumenter.rb

@@ -0,0 +1,24 @@
+module Racecar
+  ##
+  # Common API for instrumentation to standardize
+  # namespace and default payload
+  #
+  class Instrumenter
+    NAMESPACE = "racecar"
+    attr_reader :backend
+
+    def initialize(default_payload = {})
+      @default_payload = default_payload
+
+      @backend = if defined?(ActiveSupport::Notifications)
+        ActiveSupport::Notifications
+      else
+        NullInstrumenter
+      end
+    end
+
+    def instrument(event_name, payload = {}, &block)
+      @backend.instrument("#{event_name}.#{NAMESPACE}", @default_payload.merge(payload), &block)
+    end
+  end
+end

data/lib/racecar/runner.rb

@@ -53,13 +53,16 @@ module Racecar
         instrumenter: @instrumenter,
       )
 
-      instrument_payload = { consumer_class: processor.class.to_s, consumer_set: consumer }
+      instrumentation_payload = {
+        consumer_class: processor.class.to_s,
+        consumer_set: consumer
+      }
 
       # Main loop
       loop do
         break if @stop_requested
         resume_paused_partitions
-        @instrumenter.instrument("main_loop.racecar", instrument_payload) do
+        @instrumenter.instrument("main_loop", instrumentation_payload) do
           case process_method
           when :batch then
             msg_per_part = consumer.batch_poll(config.max_wait_time).group_by(&:partition)
@@ -77,7 +80,9 @@ module Racecar
       processor.deliver!
       processor.teardown
       consumer.commit
-      consumer.close
+      @instrumenter.instrument('leave_group') do
+        consumer.close
+      end
     end
 
     def stop
@@ -106,7 +111,7 @@ module Racecar
         # a value within librdkafka and is asynchronously written to proper
         # storage through auto commits.
         config.consumer << "enable.auto.offset.store=false"
-        ConsumerSet.new(config, logger)
+        ConsumerSet.new(config, logger, @instrumenter)
       end
     end
 
@@ -130,8 +135,11 @@ module Racecar
 
     def delivery_callback
       ->(delivery_report) do
-        data = {offset: delivery_report.offset, partition: delivery_report.partition}
-        @instrumenter.instrument("acknowledged_message.racecar", data)
+        payload = {
+          offset: delivery_report.offset,
+          partition: delivery_report.partition
+        }
+        @instrumenter.instrument("acknowledged_message", payload)
       end
     end
 
@@ -146,14 +154,19 @@ module Racecar
     end
 
     def process(message)
-      payload = {
+      instrumentation_payload = {
         consumer_class: processor.class.to_s,
-        topic: message.topic,
-        partition: message.partition,
-        offset: message.offset,
+        topic:          message.topic,
+        partition:      message.partition,
+        offset:         message.offset,
+        create_time:    message.timestamp,
+        key:            message.key,
+        value:          message.payload,
+        headers:        message.headers
       }
 
-      @instrumenter.instrument("process_message.racecar", payload) do
+      @instrumenter.instrument("start_process_message", instrumentation_payload)
+      @instrumenter.instrument("process_message", instrumentation_payload) do
         with_pause(message.topic, message.partition, message.offset..message.offset) do
           processor.process(Racecar::Message.new(message))
           processor.deliver!
@@ -163,16 +176,18 @@ module Racecar
     end
 
     def process_batch(messages)
-      payload = {
+      first, last = messages.first, messages.last
+      instrumentation_payload = {
         consumer_class: processor.class.to_s,
-        topic: messages.first.topic,
-        partition: messages.first.partition,
-        first_offset: messages.first.offset,
-        message_count: messages.size,
+        topic:          first.topic,
+        partition:      first.partition,
+        first_offset:   first.offset,
+        last_offset:    last.offset,
+        message_count:  messages.size
       }
 
-      @instrumenter.instrument("process_batch.racecar", payload) do
-        first, last = messages.first, messages.last
+      @instrumenter.instrument("start_process_batch", instrumentation_payload)
+      @instrumenter.instrument("process_batch", instrumentation_payload) do
         with_pause(first.topic, first.partition, first.offset..last.offset) do
           processor.process_batch(messages.map {|message| Racecar::Message.new(message) })
           processor.deliver!
@@ -204,11 +219,13 @@ module Racecar
 
       pauses.each do |topic, partitions|
         partitions.each do |partition, pause|
-          @instrumenter.instrument("pause_status.racecar", {
-            topic: topic,
-            partition: partition,
-            duration: pause.pause_duration,
-          })
+          instrumentation_payload = {
+            topic:      topic,
+            partition:  partition,
+            duration:   pause.pause_duration,
+            consumer_class: processor.class.to_s,
+          }
+          @instrumenter.instrument("pause_status", instrumentation_payload)
 
           if pause.paused? && pause.expired?
             logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"

data/lib/racecar/version.rb

@@ -1,3 +1,3 @@
 module Racecar
-  VERSION = "2.0.0.beta1"
+  VERSION = "2.0.0.beta2"
 end

data/racecar.gemspec

@@ -27,4 +27,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "timecop"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0", "< 5.0.0"
+  spec.add_development_dependency "activesupport", ">= 4.0", "< 6.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: racecar
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta1
+  version: 2.0.0.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-08 00:00:00.000000000 Z
+date: 2020-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: king_konf
@@ -101,6 +101,46 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dogstatsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 description: 
 email:
 - dschierbeck@zendesk.com
@@ -139,6 +179,8 @@ files:
 - lib/racecar/consumer_set.rb
 - lib/racecar/ctl.rb
 - lib/racecar/daemon.rb
+- lib/racecar/datadog.rb
+- lib/racecar/instrumenter.rb
 - lib/racecar/message.rb
 - lib/racecar/null_instrumenter.rb
 - lib/racecar/pause.rb