libhoney 1.18.0 → 2.0.0

data/lib/libhoney/client.rb

@@ -46,18 +46,19 @@ module Libhoney
     #   Honeycomb query engine will interpret it as representative of 10 events)
     # @param api_host [String] defaults to +API_HOST+, override to change the
     #   destination for these Honeycomb events.
-    # @param transmission [Object] transport used to actually send events. If nil (the default), will be lazily initialized with a {TransmissionClient} on first event send.
+    # @param transmission [Class, Object, nil] transport used to actually send events. If nil (the default),
+    #   will be initialized with a {TransmissionClient}. If Class--for example, {MockTransmissionClient}--will attempt
+    #   to create a new instance of that class with {TransmissionClient}'s usual parameters. If Object, checks
+    #   that the instance passed in implements the Transmission interface (responds to :add and :close). If the
+    #   check does not succeed, a no-op NullTransmission will be used instead.
     # @param block_on_send [Boolean] if more than pending_work_capacity events are written, block sending further events
     # @param block_on_responses [Boolean] if true, block if there is no thread reading from the response queue
     # @param pending_work_capacity [Fixnum] defaults to 1000. If the queue of
     #   pending events exceeds 1000, this client will start dropping events.
-    # @param proxy_config [String, Array, nil] proxy connection information
+    # @param proxy_config [String, nil] proxy connection information
     #   nil: (default, recommended) connection proxying will be determined from any http_proxy, https_proxy, and no_proxy environment
     #     variables set for the process.
     #   String: the value must be the URI for connecting to a forwarding web proxy. Must be parsable by stdlib URI.
-    #   Array: (deprecated, removal in v2.0) the value must have one and at most four elements: e.g. ['host', port, 'username', 'password'].
-    #     The assumption is that the TCP connection will be tunneled via HTTP, so the assumed scheme is 'http://'
-    #     'host' is required. 'port' is optional (default:80), unless a 'username' is included. 'password' is optional.
     # rubocop:disable Metrics/ParameterLists
     def initialize(writekey: nil,
                    dataset: nil,
@@ -74,12 +75,8 @@ module Libhoney
                    proxy_config: nil)
       # rubocop:enable Metrics/ParameterLists
       # check for insanity
-      raise Exception, 'libhoney:  max_concurrent_batches must be greater than 0' if max_concurrent_batches < 1
-      raise Exception, 'libhoney:  sample rate must be greater than 0'            if sample_rate < 1
-
-      unless Gem::Dependency.new('ruby', '>= 2.2').match?('ruby', RUBY_VERSION)
-        raise Exception, 'libhoney:  Ruby versions < 2.2 are not supported'
-      end
+      raise 'libhoney:  max_concurrent_batches must be greater than 0' if max_concurrent_batches < 1
+      raise 'libhoney:  sample rate must be greater than 0'            if sample_rate < 1
 
       @builder = Builder.new(self, nil)
 
@@ -88,17 +85,6 @@ module Libhoney
       @builder.sample_rate = sample_rate
       @builder.api_host    = api_host
 
-      @transmission = transmission
-      if !@transmission && !(writekey && dataset)
-        # if no writekey or dataset are configured, and we didn't override the
-        # transmission (e.g. to a MockTransmissionClient), that's almost
-        # certainly a misconfiguration, even though it's possible to override
-        # them on a per-event basis. So let's handle the misconfiguration
-        # early rather than potentially throwing thousands of exceptions at runtime.
-        warn "#{self.class.name}: no #{writekey ? 'dataset' : 'writekey'} configured, disabling sending events"
-        @transmission = NullTransmissionClient.new
-      end
-
       @user_agent_addition = user_agent_addition
 
       @block_on_send          = block_on_send
@@ -108,8 +94,8 @@ module Libhoney
       @max_concurrent_batches = max_concurrent_batches
       @pending_work_capacity  = pending_work_capacity
       @responses              = SizedQueue.new(2 * @pending_work_capacity)
-      @lock                   = Mutex.new
       @proxy_config           = parse_proxy_config(proxy_config)
+      @transmission           = setup_transmission(transmission, writekey, dataset)
     end
 
     attr_reader :block_on_send, :block_on_responses, :max_batch_size,
@@ -121,7 +107,7 @@ module Libhoney
 
     # Nuke the queue and wait for inflight requests to complete before returning.
     # If you set drain=false, all queued requests will be dropped on the floor.
-    def close(drain = true)
+    def close(drain = true) # rubocop:disable Style/OptionalBooleanParameter
       return @transmission.close(drain) if @transmission
 
       0
@@ -197,22 +183,6 @@ module Libhoney
     # @param event [Event] the event to send to honeycomb
     # @api private
     def send_event(event)
-      @lock.synchronize do
-        transmission_client_params = {
-          max_batch_size: @max_batch_size,
-          send_frequency: @send_frequency,
-          max_concurrent_batches: @max_concurrent_batches,
-          pending_work_capacity: @pending_work_capacity,
-          responses: @responses,
-          block_on_send: @block_on_send,
-          block_on_responses: @block_on_responses,
-          user_agent_addition: @user_agent_addition,
-          proxy_config: @proxy_config
-        }
-
-        @transmission ||= TransmissionClient.new(**transmission_client_params)
-      end
-
       @transmission.add(event)
     end
 
@@ -234,6 +204,59 @@ module Libhoney
 
     private
 
+    ##
+    # Parameters to pass to a transmission based on client config.
+    #
+    def transmission_client_params
+      {
+        max_batch_size: @max_batch_size,
+        send_frequency: @send_frequency,
+        max_concurrent_batches: @max_concurrent_batches,
+        pending_work_capacity: @pending_work_capacity,
+        responses: @responses,
+        block_on_send: @block_on_send,
+        block_on_responses: @block_on_responses,
+        user_agent_addition: @user_agent_addition,
+        proxy_config: @proxy_config
+      }
+    end
+
+    def setup_transmission(transmission, writekey, dataset)
+      # if a provided transmission can add and close, we'll assume the user
+      # has provided a working transmission with a customized configuration
+      return transmission if quacks_like_a_transmission?(transmission)
+
+      if !(writekey && dataset) # rubocop:disable Style/NegatedIf
+        # if no writekey or dataset are configured, and we didn't override the
+        # transmission (e.g. to a MockTransmissionClient), that's almost
+        # certainly a misconfiguration, even though it's possible to override
+        # them on a per-event basis. So let's handle the misconfiguration
+        # early rather than potentially throwing thousands of exceptions at runtime.
+        warn "#{self.class.name}: no #{writekey ? 'dataset' : 'writekey'} configured, disabling sending events"
+        return NullTransmissionClient.new
+      end
+
+      case transmission
+      # rubocop:disable Style/GuardClause, Style/RedundantReturn
+      when NilClass # the default value for new clients
+        return TransmissionClient.new(**transmission_client_params)
+      when Class
+        # if a class has been provided, attempt to instantiate it with parameters given to the client
+        t = transmission.new(**transmission_client_params)
+        if quacks_like_a_transmission?(t)
+          return t
+        else
+          warn "#{t.class.name}: does not appear to behave like a transmission, disabling sending events"
+          return NullTransmissionClient.new
+        end
+      end
+      # rubocop:enable Style/GuardClause, Style/RedundantReturn
+    end
+
+    def quacks_like_a_transmission?(transmission)
+      transmission.respond_to?(:add) && transmission.respond_to?(:close)
+    end
+
     # @api private
     def parse_proxy_config(config)
       case config
@@ -241,22 +264,13 @@ module Libhoney
       when String
         URI.parse(config)
       when Array
-        warn <<-WARNING
-        DEPRECATION WARNING: #{self.class.name} the proxy_config parameter will require a String value, not an Array in libhoney 2.0.
-        To resolve:
-          + recommended: set http/https_proxy environment variables, which take precedence over any option set here, then remove proxy_config parameter from client initialization
-          + set proxy_config to a String containing the forwarding proxy URI (only used if http/https_proxy are not set)
-        WARNING
-        host, port, user, password = config
-
-        parsed_config = URI::HTTP.build(host: host, port: port).tap do |uri|
-          uri.userinfo = "#{user}:#{password}" if user
-        end
-        redacted_config = parsed_config.dup.tap do |uri|
-          uri.password = 'REDACTED' unless uri.password.nil? || uri.password.empty?
-        end
-        warn "The array config given has been assumed to mean: #{redacted_config}"
-        parsed_config
+        error_message = <<~MESSAGE
+          #{self.class.name}: the optional proxy_config parameter requires a String value; Array is no longer supported.
+          To resolve:
+            + recommended: set http/https_proxy environment variables, which take precedence over any option set here, then remove proxy_config parameter from client initialization
+            + set proxy_config to a String containing the forwarding proxy URI (only used if http/https_proxy environment variables are not set)
+        MESSAGE
+        raise error_message
       end
     rescue URI::Error => e
       warn "#{self.class.name}: unable to parse proxy_config. Detail: #{e.class}: #{e.message}"

data/lib/libhoney/experimental_transmission.rb

@@ -0,0 +1,107 @@
+require 'libhoney/queueing'
+require 'libhoney/transmission'
+
+module Libhoney
+  ##
+  # An experimental variant of the standard {TransmissionClient} that uses
+  # a custom implementation of a sized queue whose pop/push methods support
+  # a timeout internally.
+  #
+  # @example Use this transmission with the Ruby Beeline
+  #   require 'libhoney/experimental_transmission'
+  #
+  #   Honeycomb.configure do |config|
+  #     config.client = Libhoney::Client.new(
+  #       writekey: ENV["HONEYCOMB_WRITE_KEY"],
+  #       dataset: ENV.fetch("HONEYCOMB_DATASET", "awesome_sauce"),
+  #       transmission: Libhoney::ExperimentalTransmissionClient
+  #     )
+  #     ...
+  #   end
+  #
+  # @api private
+  #
+  class ExperimentalTransmissionClient < TransmissionClient
+    def add(event)
+      return unless event_valid(event)
+
+      begin
+        # if block_on_send is true, never timeout the wait to enqueue an event
+        # otherwise, timeout the wait immediately and if the queue is full, we'll
+        # have a ThreadError raised because we could not add to the queue.
+        timeout = @block_on_send ? :never : 0
+        @batch_queue.enq(event, timeout)
+      rescue Libhoney::Queueing::SizedQueueWithTimeout::PushTimedOut
+        # happens if the queue was full and block_on_send = false.
+        warn "#{self.class.name}: batch queue full, dropping event." if %w[debug trace].include?(ENV['LOG_LEVEL'])
+      end
+
+      ensure_threads_running
+    end
+
+    def batch_loop
+      next_send_time = Time.now + @send_frequency
+      batched_events = Hash.new do |h, key|
+        h[key] = []
+      end
+
+      loop do
+        begin
+          # an event on the batch_queue
+          #   1. pops out and is truthy
+          #   2. gets included in the current batch
+          #   3. while waits for another event
+          while (event = @batch_queue.pop(@send_frequency))
+            key = [event.api_host, event.writekey, event.dataset]
+            batched_events[key] << event
+          end
+
+          # a nil on the batch_queue
+          #   1. pops out and is falsy
+          #   2. ends the event-popping while do..end
+          #   3. breaks the loop
+          #   4. flushes the current batch
+          #   5. ends the batch_loop
+          break
+
+        # a timeout expiration waiting for an event
+        #   1. skips the break and is rescued
+        #   2. triggers the ensure to flush the current batch
+        #   3. begins the loop again with an updated next_send_time
+        rescue Libhoney::Queueing::SizedQueueWithTimeout::PopTimedOut => e
+          warn "#{self.class.name}: ⏱ " + e.message if %w[trace].include?(ENV['LOG_LEVEL'])
+
+        # any exception occurring in this loop should not take down the actual
+        # instrumented Ruby process, so handle here and log that there is trouble
+        rescue Exception => e
+          warn "#{self.class.name}: 💥 " + e.message if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          warn e.backtrace.join("\n").to_s if ['trace'].include?(ENV['LOG_LEVEL'])
+
+        # regardless of the exception, figure out whether enough time has passed to
+        # send the current batched events, if so, send them and figure out the next send time
+        # before going back to the top of the loop
+        ensure
+          next_send_time = flush_batched_events(batched_events) if Time.now > next_send_time
+        end
+      end
+
+      # don't need to capture the next_send_time here because the batch_loop is exiting
+      # for some reason (probably transmission.close)
+      flush_batched_events(batched_events)
+    end
+
+    private
+
+    def setup_batch_queue
+      # override super()'s @batch_queue = SizedQueue.new(); use our SizedQueueWithTimeout:
+      # + block on adding events to the batch_queue when queue is full and @block_on_send is true
+      # + the queue knows how to limit size and how to time-out pushes and pops
+      @batch_queue = Libhoney::Queueing::SizedQueueWithTimeout.new(@pending_work_capacity)
+      warn "⚠️🐆 #{self.class.name} in use! It may drop data, consume all your memory, or cause skin irritation."
+    end
+
+    def build_user_agent(user_agent_addition)
+      super("(exp-transmission) #{user_agent_addition}".strip)
+    end
+  end
+end

data/lib/libhoney/log_client.rb

@@ -10,7 +10,7 @@ module Libhoney
   #       to verify what events your instrumented code is sending. Use in
   #       production is not recommended.
   class LogClient < Client
-    def initialize(*args, output: ::STDERR, verbose: false, **kwargs)
+    def initialize(*args, output: $stderr, verbose: false, **kwargs)
       super(*args,
             transmission: LogTransmissionClient.new(output: output, verbose: verbose),
             **kwargs)

data/lib/libhoney/mock_transmission.rb

@@ -7,7 +7,7 @@ module Libhoney
   #       verify what events your instrumented code is sending. Use in
   #       production is not recommended.
   class MockTransmissionClient
-    def initialize
+    def initialize(**_)
       reset
     end
 

data/lib/libhoney/null_transmission.rb

@@ -4,6 +4,8 @@ module Libhoney
   #
   # @api private
   class NullTransmissionClient
+    def initialize(**_); end
+
     def add(event); end
 
     def close(drain); end

data/lib/libhoney/queueing/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2021 Honeycomb
+Copyright (c) 2013 Avdi Grimm
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/libhoney/queueing/sized_queue_with_timeout.rb

@@ -0,0 +1,176 @@
+##
+# SizedQueueWithTimeout is copyright and licensed per the LICENSE.txt in
+# its containing subdirectory of this codebase.
+#
+module Libhoney
+  module Queueing
+    ##
+    # A queue implementation with optional size limit and optional timeouts on pop and push
+    # operations. Heavily influenced / liberally mimicking Avdi Grimm's
+    # {Tapas::Queue}[https://github.com/avdi/tapas-queue].
+    #
+    class SizedQueueWithTimeout
+      class PushTimedOut < ThreadError; end
+      class PopTimedOut < ThreadError; end
+
+      ##
+      # @param max_size [Integer, Float::INFINITY] the size limit for this queue
+      # @param options [Hash] optional dependencies to inject, primarily for testing
+      # @option options [QLock, Mutex] :lock the lock for synchronizing queue state change
+      # @option options [QCondition] :space_available_condition the condition variable
+      #   to wait/signal on for space being available in the queue; when provided, must
+      #   be accompanied by an +:item_available_condition+ and the shared +:lock+
+      # @option options [QCondition] :item_available_condition the condition variable
+      #   to wait/signal on for an item being added to the queue; when provided, must
+      #   be accompanied by an +:space_available_condition+ and the shared +:lock+
+      def initialize(max_size = Float::INFINITY, options = {})
+        @items           = []
+        @max_size        = max_size
+        @lock            = options.fetch(:lock) { QLock.new }
+        @space_available = options.fetch(:space_available_condition) { QCondition.new(@lock) }
+        @item_available  = options.fetch(:item_available_condition) { QCondition.new(@lock) }
+      end
+
+      ##
+      # Push something onto the queue.
+      #
+      # @param obj [Object] the thing to add to the queue
+      # @param timeout [Numeric, :never] how long in seconds to wait for the queue to have space available or
+      #   +:never+ to wait "forever"
+      # @param timeout_policy [#call] defaults to +-> { raise PushTimedOut }+ - a lambda/Proc/callable, what to do
+      #   when the timeout expires
+      #
+      # @raise {PushTimedOut}
+      def push(obj, timeout = :never, &timeout_policy)
+        timeout_policy ||= -> { raise PushTimedOut }
+
+        wait_for_condition(@space_available, -> { !full? }, timeout, timeout_policy) do
+          @items.push(obj)
+          @item_available.signal
+        end
+      end
+      alias enq push
+      alias << push
+
+      ##
+      # Pop something off the queue.
+      #
+      # @param timeout [Numeric, :never] how long in seconds to wait for the queue to have an item available or
+      #   +:never+ to wait "forever"
+      # @param timeout_policy [#call] defaults to +-> { raise PopTimedOut }+ - a lambda/Proc/callable, what to do
+      #   when the timeout expires
+      #
+      # @return [Object]
+      # @raise {PopTimedOut}
+      def pop(timeout = :never, &timeout_policy)
+        timeout_policy ||= -> { raise PopTimedOut }
+
+        wait_for_condition(@item_available, -> { !empty? }, timeout, timeout_policy) do
+          item = @items.shift
+          @space_available.signal unless full?
+          item
+        end
+      end
+      alias deq pop
+      alias shift pop
+
+      ##
+      # Removes all objects from the queue. They are cast into the abyss never to be seen again.
+      #
+      def clear
+        @lock.synchronize do
+          @items = []
+          @space_available.signal unless full?
+        end
+      end
+
+      private
+
+      ##
+      # Whether the queue is at capacity. Must be called with the queue's lock
+      # or the answer won't matter if you try to change state based on it.
+      #
+      # @return [true/false]
+      # @api private
+      def full?
+        @max_size <= @items.size
+      end
+
+      ##
+      # Whether the queue is empty. Must be called with the queue's lock or the
+      # answer won't matter if you try to change state based on it.
+      #
+      # @return [true/false]
+      # @api private
+      def empty?
+        @items.empty?
+      end
+
+      # a generic conditional variable wait with a timeout loop
+      #
+      # @param condition [#wait] a condition variable to wait upon.
+      # @param condition_predicate [#call] a callable (i.e. lambda or proc) that returns true/false to act
+      #   as a state tester (i.e. "is the queue currently empty?") to check on whether to keep waiting;
+      #   used to handle spurious wake ups occurring before the timeout has elapsed
+      # @param timeout [:never, Numeric] the amount of time in (seconds?) to wait, or :never to wait forever
+      # @param timeout_policy [#call] a callable, what to do when a timeout occurs? Return a default? Raise an
+      #   exception? You decide.
+      def wait_for_condition(condition, condition_predicate, timeout = :never, timeout_policy = -> {})
+        deadline = timeout == :never ? :never : trustworthy_current_time + timeout
+        @lock.synchronize do
+          loop do
+            time_remaining = timeout == :never ? nil : deadline - trustworthy_current_time
+
+            if !condition_predicate.call && time_remaining.to_f >= 0 # rubocop:disable Style/IfUnlessModifier
+              condition.wait(time_remaining)
+            end
+
+            if condition_predicate.call # rubocop:disable Style/GuardClause
+              return yield
+            elsif deadline == :never || deadline > trustworthy_current_time
+              next
+            else
+              return timeout_policy.call
+            end
+          end
+        end
+      end
+
+      # Within the context of the current process, return time from a
+      # monotonically increasing clock because for timeouts we care about
+      # elapsed time within the process, not human time.
+      #
+      # @return [Numeric]
+      def trustworthy_current_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+
+    class QCondition
+      def initialize(lock)
+        @lock = lock
+        @cv   = ConditionVariable.new
+      end
+
+      def wait(timeout = nil)
+        @cv.wait(@lock.mutex, timeout)
+      end
+
+      def signal
+        @cv.signal
+      end
+    end
+
+    class QLock
+      attr_reader :mutex
+
+      def initialize
+        @mutex = Mutex.new
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+    end
+  end
+end

data/lib/libhoney/queueing.rb

@@ -0,0 +1 @@
+require 'libhoney/queueing/sized_queue_with_timeout'