dogstatsd-ruby 4.7.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57d7660dad73ad8c0c5f37f6d6130474d255beaa467aca03f532e552d0b608e0
-  data.tar.gz: 30c8686d3045e4cfb2ca18c57390f78dbcccba1ff941deb170d89497953d5e84
+  metadata.gz: f9f9d5b8de35189b467aae9471e7dcbd8a2913bfa94a0ebea428c4088ed1bf6e
+  data.tar.gz: 5a8ec414ba8e7b97dc5ff2d720b183ae12ce3e8f32dd022889276e02fa852ef2
 SHA512:
-  metadata.gz: 99214bb827186d3b01568cc59064f7d2a8d043a4a739c7f90a2b013feeeead11ed363937c37d2c29846d101ed886c318a6369c2d5a57292b0c058db81c708813
-  data.tar.gz: c82e6c5e9668804688fd308e091398e05ada88dac8ea78a678ee2da671eaf297ffd95997efaf376506632f65e88b4096a6a9b2590667f772f19c6b020e238827
+  metadata.gz: 0a89bc622a5fcb9c5f2376aa0573a3af843b3ce4952505b7d2076ef6f1ee58f6a8d2ef9b2498f919327bae37f05d43970c975a14a654019bef21cf26153d0080
+  data.tar.gz: 7efd1aaf9d2b4a6795422ad012da1226c43ed3c85aca1242d8c7defee5ef1d1e5decc34d0a5ce767f824cd8f3ad84182ea0b8c56c2fb0e5f0d1bb28b79cc52a8

data/README.md

@@ -71,9 +71,22 @@ After the client is created, you can start sending events to your Datadog Event
 
 After the client is created, you can start sending Service Checks to Datadog. See the dedicated [Service Check Submission: DogStatsD documentation](https://docs.datadoghq.com/developers/service_checks/dogstatsd_service_checks_submission/?tab=ruby) to see how to submit a Service Check to Datadog.
 
+### Maximum packets size in high-throughput scenarios
+
+In order to have the most efficient use of this library in high-throughput scenarios,
+default values for the maximum packets size have already been set for both UDS (8192 bytes)
+and UDP (1432 bytes) in order to have the best usage of the underlying network.
+However, if you perfectly know your network and you know that a different value for the maximum packets
+size should be used, you can set it with the parameter `buffer_max_payload_size`. Example:
+
+```ruby
+# Create a DogStatsD client instance.
+statsd = Datadog::Statsd.new('localhost', 8125, buffer_max_payload_size: 4096)
+```
+
 ## Credits
 
-dogstatsd-ruby is forked from Rien Henrichs [original Statsd
+dogstatsd-ruby is forked from Rein Henrichs [original Statsd
 client](https://github.com/reinh/statsd).
 
 Copyright (c) 2011 Rein Henrichs. See LICENSE.txt for

data/lib/datadog/statsd.rb

@@ -1,10 +1,14 @@
 # frozen_string_literal: true
 require 'socket'
 
+require_relative 'statsd/version'
 require_relative 'statsd/telemetry'
 require_relative 'statsd/udp_connection'
 require_relative 'statsd/uds_connection'
-require_relative 'statsd/batch'
+require_relative 'statsd/message_buffer'
+require_relative 'statsd/serialization'
+require_relative 'statsd/sender'
+require_relative 'statsd/forwarder'
 
 # = Datadog::Statsd: A DogStatsd client (https://www.datadoghq.com)
 #
@@ -24,31 +28,17 @@ require_relative 'statsd/batch'
 #   statsd = Datadog::Statsd.new 'localhost', 8125, tags: 'tag1:true'
 module Datadog
   class Statsd
-    # Create a dictionary to assign a key to every parameter's name, except for tags (treated differently)
-    # Goal: Simple and fast to add some other parameters
-    OPTS_KEYS = {
-      date_happened:    :d,
-      hostname:         :h,
-      aggregation_key:  :k,
-      priority:         :p,
-      source_type_name: :s,
-      alert_type:       :t,
-    }.freeze
-
-    # Service check options
-    SC_OPT_KEYS = {
-      timestamp: 'd:',
-      hostname:  'h:',
-      tags:      '#',
-      message:   'm:',
-    }.freeze
+    class Error < StandardError
+    end
 
     OK       = 0
     WARNING  = 1
     CRITICAL = 2
     UNKNOWN  = 3
 
-    DEFAULT_BUFFER_SIZE = 8 * 1_024
+    UDP_DEFAULT_BUFFER_SIZE = 1_432
+    UDS_DEFAULT_BUFFER_SIZE = 8_192
+    DEFAULT_BUFFER_POOL_SIZE = Float::INFINITY
     MAX_EVENT_SIZE = 8 * 1_024
     # minimum flush interval for the telemetry in seconds
     DEFAULT_TELEMETRY_FLUSH_INTERVAL = 10
@@ -59,80 +49,68 @@ module Datadog
     DISTRIBUTION_TYPE = 'd'
     TIMING_TYPE = 'ms'
     SET_TYPE = 's'
-    VERSION = '4.7.0'
 
     # A namespace to prepend to all statsd calls. Defaults to no namespace.
     attr_reader :namespace
 
     # Global tags to be added to every statsd call. Defaults to no tags.
-    attr_reader :tags
-
-    # Buffer containing the statsd message before they are sent in batch
-    attr_reader :buffer
-
-    # Maximum buffer size in bytes before it is flushed
-    attr_reader :max_buffer_bytes
+    def tags
+      serializer.global_tags
+    end
 
     # Default sample rate
     attr_reader :sample_rate
 
-    # Connection
-    attr_reader :connection
-
     # @param [String] host your statsd host
     # @param [Integer] port your statsd port
     # @option [String] namespace set a namespace to be prepended to every metric name
     # @option [Array<String>|Hash] tags tags to be added to every metric
     # @option [Logger] logger for debugging
-    # @option [Integer] max_buffer_bytes max bytes to buffer when using #batch
+    # @option [Integer] buffer_max_payload_size max bytes to buffer
+    # @option [Integer] buffer_max_pool_size max messages to buffer
     # @option [String] socket_path unix socket path
     # @option [Float] default sample rate if not overridden
     def initialize(
       host = nil,
       port = nil,
+      socket_path: nil,
+
       namespace: nil,
       tags: nil,
-      max_buffer_bytes: DEFAULT_BUFFER_SIZE,
-      socket_path: nil,
-      logger: nil,
       sample_rate: nil,
-      disable_telemetry: false,
-      telemetry_flush_interval: DEFAULT_TELEMETRY_FLUSH_INTERVAL
-    )
-      unless tags.nil? || tags.is_a?(Array) || tags.is_a?(Hash)
-        raise ArgumentError, 'tags must be a Array<String> or a Hash'
-      end
 
-      tags = tag_hash_to_array(tags) if tags.is_a?(Hash)
-      @tags = (tags || []).compact.map! do |tag|
-        escape_tag_content(tag)
-      end
+      buffer_max_payload_size: nil,
+      buffer_max_pool_size: nil,
+      buffer_overflowing_stategy: :drop,
 
-      # append the entity id to tags if DD_ENTITY_ID env var is not nil
-      unless ENV.fetch('DD_ENTITY_ID', nil).nil?
-        dd_entity = escape_tag_content(ENV.fetch('DD_ENTITY_ID', nil))
-        @tags << 'dd.internal.entity_id:' + dd_entity
-      end
-
-      # init telemetry
-      transport_type = socket_path.nil? ? 'udp': 'uds'
-      telemetry_tags = (["client:ruby", "client_version:#{VERSION}", "client_transport:#{transport_type}"] + @tags).join(COMMA).freeze
-      @telemetry = Telemetry.new(disable_telemetry, telemetry_tags, telemetry_flush_interval)
+      logger: nil,
 
-      if socket_path.nil?
-        @connection = UDPConnection.new(host, port, logger, @telemetry)
-      else
-        @connection = UDSConnection.new(socket_path, logger, @telemetry)
+      telemetry_enable: true,
+      telemetry_flush_interval: DEFAULT_TELEMETRY_FLUSH_INTERVAL
+    )
+      unless tags.nil? || tags.is_a?(Array) || tags.is_a?(Hash)
+        raise ArgumentError, 'tags must be an array of string tags or a Hash'
       end
-      @logger = logger
 
       @namespace = namespace
       @prefix = @namespace ? "#{@namespace}.".freeze : nil
-
+      @serializer = Serialization::Serializer.new(prefix: @prefix, global_tags: tags)
       @sample_rate = sample_rate
 
-      # we reduce max_buffer_bytes by a the rough estimate of the telemetry payload
-      @batch = Batch.new(@connection, (max_buffer_bytes - @telemetry.estimate_max_size))
+      @forwarder = Forwarder.new(
+        host: host,
+        port: port,
+        socket_path: socket_path,
+
+        global_tags: tags,
+        logger: logger,
+
+        buffer_max_payload_size: buffer_max_payload_size,
+        buffer_max_pool_size: buffer_max_pool_size,
+        buffer_overflowing_stategy: buffer_overflowing_stategy,
+
+        telemetry_flush_interval: telemetry_enable ? telemetry_flush_interval : nil,
+      )
     end
 
     # yield a new instance to a block and close it when done
@@ -259,20 +237,10 @@ module Datadog
     #   $statsd.time('account.activate') { @account.activate! }
     def time(stat, opts = EMPTY_OPTIONS)
       opts = { sample_rate: opts } if opts.is_a?(Numeric)
-      start = if PROCESS_TIME_SUPPORTED
-                Process.clock_gettime(Process::CLOCK_MONOTONIC) # uncovered
-              else
-                Time.now.to_f # uncovered
-              end
+      start = now
       yield
     ensure
-      finished =  if PROCESS_TIME_SUPPORTED
-                    Process.clock_gettime(Process::CLOCK_MONOTONIC) # uncovered
-                  else
-                    Time.now.to_f # uncovered
-                  end
-
-      timing(stat, ((finished - start) * 1000).round, opts)
+      timing(stat, ((now - start) * 1000).round, opts)
     end
 
     # Sends a value to be tracked as a set to the statsd server.
@@ -301,8 +269,9 @@ module Datadog
     # @example Report a critical service check status
     #   $statsd.service_check('my.service.check', Statsd::CRITICAL, :tags=>['urgent'])
     def service_check(name, status, opts = EMPTY_OPTIONS)
-      @telemetry.service_checks += 1
-      send_stat(format_service_check(name, status, opts))
+      telemetry.sent(service_checks: 1) if telemetry
+
+      forwarder.send_message(serializer.to_service_check(name, status, opts))
     end
 
     # This end point allows you to post events to the stream. You can tag them, set priority and even aggregate them with other events.
@@ -320,178 +289,77 @@ module Datadog
     # @option opts [String, nil] :priority ('normal') Can be "normal" or "low"
     # @option opts [String, nil] :source_type_name (nil) Assign a source type to the event
     # @option opts [String, nil] :alert_type ('info') Can be "error", "warning", "info" or "success".
+    # @option opts [Boolean, false] :truncate_if_too_long (false) Truncate the event if it is too long
     # @option opts [Array<String>] :tags tags to be added to every metric
     # @example Report an awful event:
     #   $statsd.event('Something terrible happened', 'The end is near if we do nothing', :alert_type=>'warning', :tags=>['end_of_times','urgent'])
     def event(title, text, opts = EMPTY_OPTIONS)
-      @telemetry.events += 1
-      send_stat(format_event(title, text, opts))
-    end
+      telemetry.sent(events: 1) if telemetry
 
-    # Send several metrics in the same UDP Packet
-    # They will be buffered and flushed when the block finishes
-    #
-    # @example Send several metrics in one packet:
-    #   $statsd.batch do |s|
-    #      s.gauge('users.online',156)
-    #      s.increment('page.views')
-    #    end
-    def batch
-      @batch.open do
-        yield self
-      end
+      forwarder.send_message(serializer.to_event(title, text, opts))
     end
 
     # Close the underlying socket
     def close
-      @connection.close
+      forwarder.close
     end
 
-    private
-
-    NEW_LINE = "\n"
-    ESC_NEW_LINE = '\n'
-    COMMA = ','
-    PIPE = '|'
-    DOT = '.'
-    DOUBLE_COLON = '::'
-    UNDERSCORE = '_'
-    PROCESS_TIME_SUPPORTED = (RUBY_VERSION >= '2.1.0')
-    EMPTY_OPTIONS = {}.freeze
-
-    private_constant :NEW_LINE, :ESC_NEW_LINE, :COMMA, :PIPE, :DOT,
-      :DOUBLE_COLON, :UNDERSCORE, :EMPTY_OPTIONS
-
-    def format_service_check(name, status, opts = EMPTY_OPTIONS)
-      sc_string = "_sc|#{name}|#{status}".dup
-
-      SC_OPT_KEYS.each do |key, shorthand_key|
-        next unless opts[key]
-
-        if key == :tags
-          if tags_string = tags_as_string(opts)
-            sc_string << "|##{tags_string}"
-          end
-        elsif key == :message
-          message = remove_pipes(opts[:message])
-          escaped_message = escape_service_check_message(message)
-          sc_string << "|m:#{escaped_message}"
-        else
-          if key == :timestamp && opts[key].is_a?(Integer)
-            value = opts[key]
-          else
-            value = remove_pipes(opts[key])
-          end
-          sc_string << "|#{shorthand_key}#{value}"
-        end
-      end
-      sc_string
+    def sync_with_outbound_io
+      forwarder.sync_with_outbound_io
     end
 
-    def format_event(title, text, opts = EMPTY_OPTIONS)
-      escaped_title = escape_event_content(title)
-      escaped_text = escape_event_content(text)
-      event_string_data = "_e{#{escaped_title.bytesize},#{escaped_text.bytesize}}:#{escaped_title}|#{escaped_text}".dup
-
-      # We construct the string to be sent by adding '|key:value' parts to it when needed
-      # All pipes ('|') in the metadata are removed. Title and Text can keep theirs
-      OPTS_KEYS.each do |key, shorthand_key|
-        if key != :tags && opts[key]
-          # :date_happened is the only key where the value is an Integer
-          # To not break backwards compatibility, we still accept a String
-          if key == :date_happened && opts[key].is_a?(Integer)
-              value = opts[key]
-          # All other keys only have String values
-          else
-              value = remove_pipes(opts[key])
-          end
-          event_string_data << "|#{shorthand_key}:#{value}"
-        end
-      end
-
-      # Tags are joined and added as last part to the string to be sent
-      if tags_string = tags_as_string(opts)
-        event_string_data << "|##{tags_string}"
-      end
-
-      if event_string_data.bytesize > MAX_EVENT_SIZE
-        raise "Event #{title} payload is too big (more that 8KB), event discarded"
-      end
-      event_string_data
+    # Flush the buffer into the connection
+    def flush(flush_telemetry: false, sync: false)
+      forwarder.flush(flush_telemetry: flush_telemetry, sync: sync)
     end
 
-    def tags_as_string(opts)
-      if tag_arr = opts[:tags]
-        tag_arr = tag_hash_to_array(tag_arr) if tag_arr.is_a?(Hash)
-        tag_arr = tag_arr.map do |tag|
-          escape_tag_content(tag)
-        end
-        tag_arr = tags + tag_arr # @tags are normalized when set, so not need to normalize them again
-      else
-        tag_arr = tags
-      end
-      tag_arr.join(COMMA) unless tag_arr.empty?
+    def telemetry
+      forwarder.telemetry
     end
 
-    def tag_hash_to_array(tag_hash)
-      tag_hash.to_a.map do |pair|
-        pair.compact.join(':')
-      end
+    def host
+      forwarder.host
     end
 
-    def escape_event_content(message)
-      message.gsub(NEW_LINE, ESC_NEW_LINE)
+    def port
+      forwarder.port
     end
 
-    def escape_tag_content(tag)
-      tag = remove_pipes(tag.to_s)
-      tag.delete!(COMMA)
-      tag
+    def socket_path
+      forwarder.socket_path
     end
 
-    def remove_pipes(message)
-      message.delete(PIPE)
+    def transport_type
+      forwarder.transport_type
     end
 
-    def escape_service_check_message(message)
-      escape_event_content(message).gsub('m:', 'm\:')
+    private
+    attr_reader :serializer
+    attr_reader :forwarder
+
+    PROCESS_TIME_SUPPORTED = (RUBY_VERSION >= '2.1.0')
+    EMPTY_OPTIONS = {}.freeze
+
+    if PROCESS_TIME_SUPPORTED
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    else
+      def now
+        Time.now.to_f
+      end
     end
 
     def send_stats(stat, delta, type, opts = EMPTY_OPTIONS)
-      @telemetry.metrics += 1
+      telemetry.sent(metrics: 1) if telemetry
+
       sample_rate = opts[:sample_rate] || @sample_rate || 1
+
       if sample_rate == 1 || rand <= sample_rate
-        full_stat = ''.dup
-        full_stat << @prefix if @prefix
-
-        stat = stat.is_a?(String) ? stat.dup : stat.to_s
-        # Replace Ruby module scoping with '.' and reserved chars (: | @) with underscores.
-        stat.gsub!(DOUBLE_COLON, DOT)
-        stat.tr!(':|@', UNDERSCORE)
-        full_stat << stat
-
-        full_stat << ':'
-        full_stat << delta.to_s
-        full_stat << PIPE
-        full_stat << type
-
-        unless sample_rate == 1
-          full_stat << PIPE
-          full_stat << '@'
-          full_stat << sample_rate.to_s
-        end
-
-        if tags_string = tags_as_string(opts)
-          full_stat << PIPE
-          full_stat << '#'
-          full_stat << tags_string
-        end
-        send_stat(full_stat)
-      end
-    end
+        full_stat = serializer.to_stat(stat, delta, type, tags: opts[:tags], sample_rate: sample_rate)
 
-    def send_stat(message)
-      @batch.open? ? @batch.add(message) : @connection.write(message)
+        forwarder.send_message(full_stat)
+      end
     end
   end
 end