libhoney 1.17.0 → 1.21.0

data/lib/libhoney/queueing/sized_queue_with_timeout.rb

@@ -0,0 +1,176 @@
+##
+# SizedQueueWithTimeout is copyright and licensed per the LICENSE.txt in
+# its containing subdirectory of this codebase.
+#
+module Libhoney
+  module Queueing
+    ##
+    # A queue implementation with optional size limit and optional timeouts on pop and push
+    # operations. Heavily influenced / liberally mimicking Avdi Grimm's
+    # {Tapas::Queue}[https://github.com/avdi/tapas-queue].
+    #
+    class SizedQueueWithTimeout
+      class PushTimedOut < ThreadError; end
+      class PopTimedOut < ThreadError; end
+
+      ##
+      # @param max_size [Integer, Float::INFINITY] the size limit for this queue
+      # @param options [Hash] optional dependencies to inject, primarily for testing
+      # @option options [QLock, Mutex] :lock the lock for synchronizing queue state change
+      # @option options [QCondition] :space_available_condition the condition variable
+      #   to wait/signal on for space being available in the queue; when provided, must
+      #   be accompanied by an +:item_available_condition+ and the shared +:lock+
+      # @option options [QCondition] :item_available_condition the condition variable
+      #   to wait/signal on for an item being added to the queue; when provided, must
+      #   be accompanied by an +:space_available_condition+ and the shared +:lock+
+      def initialize(max_size = Float::INFINITY, options = {})
+        @items           = []
+        @max_size        = max_size
+        @lock            = options.fetch(:lock) { QLock.new }
+        @space_available = options.fetch(:space_available_condition) { QCondition.new(@lock) }
+        @item_available  = options.fetch(:item_available_condition) { QCondition.new(@lock) }
+      end
+
+      ##
+      # Push something onto the queue.
+      #
+      # @param obj [Object] the thing to add to the queue
+      # @param timeout [Numeric, :never] how long in seconds to wait for the queue to have space available or
+      #   +:never+ to wait "forever"
+      # @param timeout_policy [#call] defaults to +-> { raise PushTimedOut }+ - a lambda/Proc/callable, what to do
+      #   when the timeout expires
+      #
+      # @raise {PushTimedOut}
+      def push(obj, timeout = :never, &timeout_policy)
+        timeout_policy ||= -> { raise PushTimedOut }
+
+        wait_for_condition(@space_available, -> { !full? }, timeout, timeout_policy) do
+          @items.push(obj)
+          @item_available.signal
+        end
+      end
+      alias enq push
+      alias << push
+
+      ##
+      # Pop something off the queue.
+      #
+      # @param timeout [Numeric, :never] how long in seconds to wait for the queue to have an item available or
+      #   +:never+ to wait "forever"
+      # @param timeout_policy [#call] defaults to +-> { raise PopTimedOut }+ - a lambda/Proc/callable, what to do
+      #   when the timeout expires
+      #
+      # @return [Object]
+      # @raise {PopTimedOut}
+      def pop(timeout = :never, &timeout_policy)
+        timeout_policy ||= -> { raise PopTimedOut }
+
+        wait_for_condition(@item_available, -> { !empty? }, timeout, timeout_policy) do
+          item = @items.shift
+          @space_available.signal unless full?
+          item
+        end
+      end
+      alias deq pop
+      alias shift pop
+
+      ##
+      # Removes all objects from the queue. They are cast into the abyss never to be seen again.
+      #
+      def clear
+        @lock.synchronize do
+          @items = []
+          @space_available.signal unless full?
+        end
+      end
+
+      private
+
+      ##
+      # Whether the queue is at capacity. Must be called with the queue's lock
+      # or the answer won't matter if you try to change state based on it.
+      #
+      # @return [true/false]
+      # @api private
+      def full?
+        @max_size <= @items.size
+      end
+
+      ##
+      # Whether the queue is empty. Must be called with the queue's lock or the
+      # answer won't matter if you try to change state based on it.
+      #
+      # @return [true/false]
+      # @api private
+      def empty?
+        @items.empty?
+      end
+
+      # a generic conditional variable wait with a timeout loop
+      #
+      # @param condition [#wait] a condition variable to wait upon.
+      # @param condition_predicate [#call] a callable (i.e. lambda or proc) that returns true/false to act
+      #   as a state tester (i.e. "is the queue currently empty?") to check on whether to keep waiting;
+      #   used to handle spurious wake ups occurring before the timeout has elapsed
+      # @param timeout [:never, Numeric] the amount of time in (seconds?) to wait, or :never to wait forever
+      # @param timeout_policy [#call] a callable, what to do when a timeout occurs? Return a default? Raise an
+      #   exception? You decide.
+      def wait_for_condition(condition, condition_predicate, timeout = :never, timeout_policy = -> { nil })
+        deadline = timeout == :never ? :never : trustworthy_current_time + timeout
+        @lock.synchronize do
+          loop do
+            time_remaining = timeout == :never ? nil : deadline - trustworthy_current_time
+
+            if !condition_predicate.call && time_remaining.to_f >= 0 # rubocop:disable Style/IfUnlessModifier
+              condition.wait(time_remaining)
+            end
+
+            if condition_predicate.call # rubocop:disable Style/GuardClause
+              return yield
+            elsif deadline == :never || deadline > trustworthy_current_time
+              next
+            else
+              return timeout_policy.call
+            end
+          end
+        end
+      end
+
+      # Within the context of the current process, return time from a
+      # monotonically increasing clock because for timeouts we care about
+      # elapsed time within the process, not human time.
+      #
+      # @return [Numeric]
+      def trustworthy_current_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+
+    class QCondition
+      def initialize(lock)
+        @lock = lock
+        @cv   = ConditionVariable.new
+      end
+
+      def wait(timeout = nil)
+        @cv.wait(@lock.mutex, timeout)
+      end
+
+      def signal
+        @cv.signal
+      end
+    end
+
+    class QLock
+      attr_reader :mutex
+
+      def initialize
+        @mutex = Mutex.new
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+    end
+  end
+end

data/lib/libhoney/queueing.rb

@@ -0,0 +1 @@
+require 'libhoney/queueing/sized_queue_with_timeout'

data/lib/libhoney/response.rb

@@ -1,7 +1,15 @@
-require 'http'
+require 'http/response/status'
 
 module Libhoney
   class Response
+    # The response status from HTTP calls to a Honeycomb API endpoint.
+    #
+    # For most of the life of this client, this response object has been
+    # a pass-through to the underlying HTTP library's response object.
+    # This class in the Libhoney namespace now owns the interface for
+    # API responses.
+    class Status < HTTP::Response::Status; end
+
     attr_accessor :duration, :status_code, :metadata, :error
 
     def initialize(duration: 0,
@@ -9,7 +17,7 @@ module Libhoney
                    metadata: nil,
                    error: nil)
       @duration    = duration
-      @status_code = HTTP::Response::Status.new(status_code)
+      @status_code = Status.new(status_code)
       @metadata    = metadata
       @error       = error
     end

data/lib/libhoney/transmission.rb

@@ -1,3 +1,5 @@
+require 'addressable/uri'
+require 'excon'
 require 'json'
 require 'timeout'
 require 'libhoney/response'
@@ -34,9 +36,9 @@ module Libhoney
       @send_queue   = Queue.new
       @threads      = []
       @lock         = Mutex.new
-      # use a SizedQueue so the producer will block on adding to the batch_queue when @block_on_send is true
-      @batch_queue  = SizedQueue.new(@pending_work_capacity)
       @batch_thread = nil
+
+      setup_batch_queue
     end
 
     def add(event)
@@ -51,26 +53,6 @@ module Libhoney
       ensure_threads_running
     end
 
-    def event_valid(event)
-      invalid = []
-      invalid.push('api host') if event.api_host.nil? || event.api_host.empty?
-      invalid.push('write key') if event.writekey.nil? || event.writekey.empty?
-      invalid.push('dataset') if event.dataset.nil? || event.dataset.empty?
-
-      unless invalid.empty?
-        e = StandardError.new("#{self.class.name}: nil or empty required fields (#{invalid.join(', ')})"\
-          '. Will not attempt to send.')
-        Response.new(error: e).tap do |error_response|
-          error_response.metadata = event.metadata
-          enqueue_response(error_response)
-        end
-
-        return false
-      end
-
-      true
-    end
-
     def send_loop
       http_clients = build_http_clients
 
@@ -93,7 +75,7 @@ module Libhoney
           }
 
           response = http.post(
-            "/1/batch/#{Addressable::URI.escape(dataset)}",
+            path: "/1/batch/#{Addressable::URI.escape(dataset)}",
             body: body,
             headers: headers
           )
@@ -103,6 +85,8 @@ module Libhoney
           # because this is effectively the top-level exception handler for the
           # sender threads, and we don't want those threads to die (leaving
           # nothing consuming the queue).
+          warn "#{self.class.name}: 💥 " + e.message if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          warn e.backtrace.join("\n").to_s if ['trace'].include?(ENV['LOG_LEVEL'])
           begin
             batch.each do |event|
               # nil events in the batch should already have had an error
@@ -129,23 +113,31 @@ module Libhoney
     end
 
     def close(drain)
-      # if drain is false, clear the remaining unprocessed events from the queue
-      unless drain
-        @batch_queue.clear
-        @send_queue.clear
-      end
+      @lock.synchronize do
+        # if drain is false, clear the remaining unprocessed events from the queue
+        if drain
+          warn "#{self.class.name} - close: draining events" if %w[debug trace].include?(ENV['LOG_LEVEL'])
+        else
+          warn "#{self.class.name} - close: deleting unsent events" if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          @batch_queue.clear
+          @send_queue.clear
+        end
 
-      @batch_queue.enq(nil)
-      @batch_thread.join unless @batch_thread.nil?
+        @batch_queue.enq(nil)
+        if @batch_thread.nil?
+        else
+          @batch_thread.join(1.0) # limit the amount of time we'll wait for the thread to end
+        end
 
-      # send @threads.length number of nils so each thread will fall out of send_loop
-      @threads.length.times { @send_queue << nil }
+        # send @threads.length number of nils so each thread will fall out of send_loop
+        @threads.length.times { @send_queue << nil }
 
-      @threads.each(&:join)
-      @threads = []
+        @threads.each(&:join)
+        @threads = []
+      end
 
       enqueue_response(nil)
-
+      warn "#{self.class.name} - close: close complete" if %w[debug trace].include?(ENV['LOG_LEVEL'])
       0
     end
 
@@ -157,25 +149,79 @@ module Libhoney
 
       loop do
         begin
+          # a timeout expiration waiting for an event
+          #   1. interrupts only when thread is in a blocking state (waiting for pop)
+          #   2. exception skips the break and is rescued
+          #   3. triggers the ensure to flush the current batch
+          #   3. begins the loop again with an updated next_send_time
           Thread.handle_interrupt(Timeout::Error => :on_blocking) do
+            # an event on the batch_queue
+            #   1. pops out and is truthy
+            #   2. gets included in the current batch
+            #   3. while waits for another event
             while (event = Timeout.timeout(@send_frequency) { @batch_queue.pop })
               key = [event.api_host, event.writekey, event.dataset]
               batched_events[key] << event
             end
           end
 
+          # a nil on the batch_queue
+          #   1. pops out and is falsy
+          #   2. ends the event-popping while do..end
+          #   3. breaks the loop
+          #   4. flushes the current batch
+          #   5. ends the batch_loop
           break
-        rescue Exception
+        rescue Timeout::Error
+          # Timeout::Error happens when there is nothing to pop from the batch_queue.
+          # We rescue it here to avoid spamming the logs with "execution expired" errors.
+        rescue Exception => e
+          warn "#{self.class.name}: 💥 " + e.message if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          warn e.backtrace.join("\n").to_s if ['trace'].include?(ENV['LOG_LEVEL'])
+
+        # regardless of the exception, figure out whether enough time has passed to
+        # send the current batched events, if so, send them and figure out the next send time
+        # before going back to the top of the loop
         ensure
           next_send_time = flush_batched_events(batched_events) if Time.now > next_send_time
         end
       end
 
+      # don't need to capture the next_send_time here because the batch_loop is exiting
+      # for some reason (probably transmission.close)
       flush_batched_events(batched_events)
     end
 
     private
 
+    def setup_batch_queue
+      # use a SizedQueue so the producer will block on adding to the batch_queue when @block_on_send is true
+      @batch_queue = SizedQueue.new(@pending_work_capacity)
+    end
+
+    REQUIRED_EVENT_FIELDS = %i[api_host writekey dataset].freeze
+
+    def event_valid(event)
+      missing_required_fields = REQUIRED_EVENT_FIELDS.select do |required_field|
+        event.public_send(required_field).nil? || event.public_send(required_field).empty?
+      end
+
+      if missing_required_fields.empty?
+        true
+      else
+        enqueue_response(
+          Response.new(
+            metadata: event.metadata,
+            error: StandardError.new(
+              "#{self.class.name}: nil or empty required fields (#{missing_required_fields.join(', ')})"\
+              '. Will not attempt to send.'
+            )
+          )
+        )
+        false
+      end
+    end
+
     ##
     # Enqueues a response to the responses queue suppressing ThreadError when
     # there is no space left on the queue and we are not blocking on response
@@ -186,15 +232,36 @@ module Libhoney
     end
 
     def process_response(http_response, before, batch)
-      index = 0
-      http_response.parse.each do |event|
-        index += 1 while batch[index].nil? && index < batch.size
-        break unless (batched_event = batch[index])
-
-        Response.new(status_code: event['status']).tap do |response|
-          response.duration = Time.now - before
-          response.metadata = batched_event.metadata
-          enqueue_response(response)
+      if http_response.status == 200
+        index = 0
+        JSON.parse(http_response.body).each do |event|
+          index += 1 while batch[index].nil? && index < batch.size
+          break unless (batched_event = batch[index])
+
+          enqueue_response(
+            Response.new(
+              status_code: event['status'],
+              duration: (Time.now - before),
+              metadata: batched_event.metadata
+            )
+          )
+        end
+      else
+        error = JSON.parse(http_response.body)['error']
+        if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          warn "#{self.class.name}: error sending data to Honeycomb - #{http_response.status} #{error}"
+        end
+        batch.each do |batched_event|
+          next unless batched_event # skip nils enqueued from serialization errors
+
+          enqueue_response(
+            Response.new(
+              status_code: http_response.status, # single error from API applied to all events sent in batch
+              duration: (Time.now - before),
+              metadata: batched_event.metadata,
+              error: RuntimeError.new(error)
+            )
+          )
         end
       end
     end
@@ -230,9 +297,10 @@ module Libhoney
     end
 
     def build_user_agent(user_agent_addition)
-      ua = "libhoney-rb/#{VERSION}"
-      ua << " #{user_agent_addition}" if user_agent_addition
-      ua
+      "libhoney-rb/#{VERSION}"
+        .concat(" #{user_agent_addition}")
+        .strip # remove trailing spaces if addition was empty
+        .concat(" Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM})")
     end
 
     def ensure_threads_running
@@ -256,14 +324,19 @@ module Libhoney
 
     def build_http_clients
       Hash.new do |h, api_host|
-        client = HTTP.timeout(connect: @send_timeout, write: @send_timeout, read: @send_timeout)
-                     .persistent(api_host)
-                     .headers(
-                       'User-Agent' => @user_agent,
-                       'Content-Type' => 'application/json'
-                     )
-
-        client = client.via(*@proxy_config) unless @proxy_config.nil?
+        client = ::Excon.new(
+          api_host,
+          persistent: true,
+          read_timeout: @send_timeout,
+          write_timeout: @send_timeout,
+          connect_timeout: @send_timeout,
+          proxy: @proxy_config,
+          headers: {
+            'User-Agent' => @user_agent,
+            'Content-Type' => 'application/json'
+          }
+        )
+
         h[api_host] = client
       end
     end

data/lib/libhoney/version.rb

@@ -1,3 +1,3 @@
 module Libhoney
-  VERSION = '1.17.0'.freeze
+  VERSION = '1.21.0'.freeze
 end

data/libhoney.gemspec

@@ -24,15 +24,18 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bump', '~> 0.5'
   spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'lockstep'
   spec.add_development_dependency 'minitest', '~> 5.0'
-  spec.add_development_dependency 'rake', '~> 12.3'
+  spec.add_development_dependency 'minitest-reporters'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rubocop', '< 0.69'
   spec.add_development_dependency 'sinatra'
   spec.add_development_dependency 'sinatra-contrib'
-  spec.add_development_dependency 'spy', '1.0.0'
+  spec.add_development_dependency 'spy', '~> 1.0'
   spec.add_development_dependency 'webmock', '~> 3.4'
   spec.add_development_dependency 'yard'
   spec.add_development_dependency 'yardstick', '~> 0.9'
   spec.add_dependency 'addressable', '~> 2.0'
-  spec.add_dependency 'http', '>= 2.0', '< 5.0'
+  spec.add_dependency 'excon'
+  spec.add_dependency 'http', '>= 2.0', '< 6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libhoney
 version: !ruby/object:Gem::Version
-  version: 1.17.0
+  version: 1.21.0
 platform: ruby
 authors:
 - The Honeycomb.io Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-04 00:00:00.000000000 Z
+date: 2021-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: lockstep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -52,20 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -112,16 +140,16 @@ dependencies:
   name: spy
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
@@ -187,7 +229,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -197,7 +239,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 description: Ruby gem for sending data to Honeycomb
 email: support@honeycomb.io
 executables: []
@@ -208,27 +250,45 @@ files:
 - ".circleci/setup-rubygems.sh"
 - ".editorconfig"
 - ".github/CODEOWNERS"
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/ISSUE_TEMPLATE/question-discussion.md"
+- ".github/ISSUE_TEMPLATE/security-vulnerability-report.md"
+- ".github/PULL_REQUEST_TEMPLATE.md"
+- ".github/dependabot.yml"
+- ".github/workflows/add-to-project.yml"
+- ".github/workflows/apply-labels.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
 - CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE
+- OSSMETADATA
 - README.md
+- RELEASING.md
 - Rakefile
+- SECURITY.md
+- SUPPORT.md
 - example/factorial.rb
 - lib/libhoney.rb
 - lib/libhoney/builder.rb
 - lib/libhoney/cleaner.rb
 - lib/libhoney/client.rb
 - lib/libhoney/event.rb
+- lib/libhoney/experimental_transmission.rb
 - lib/libhoney/log_client.rb
 - lib/libhoney/log_transmission.rb
 - lib/libhoney/mock_transmission.rb
 - lib/libhoney/null_client.rb
 - lib/libhoney/null_transmission.rb
+- lib/libhoney/queueing.rb
+- lib/libhoney/queueing/LICENSE.txt
+- lib/libhoney/queueing/sized_queue_with_timeout.rb
 - lib/libhoney/response.rb
 - lib/libhoney/test_client.rb
 - lib/libhoney/transmission.rb
@@ -253,7 +313,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: send data to Honeycomb