libhoney 1.18.0 → 2.0.0

data/lib/libhoney/transmission.rb

@@ -36,9 +36,9 @@ module Libhoney
       @send_queue   = Queue.new
       @threads      = []
       @lock         = Mutex.new
-      # use a SizedQueue so the producer will block on adding to the batch_queue when @block_on_send is true
-      @batch_queue  = SizedQueue.new(@pending_work_capacity)
       @batch_thread = nil
+
+      setup_batch_queue
     end
 
     def add(event)
@@ -53,26 +53,6 @@ module Libhoney
       ensure_threads_running
     end
 
-    def event_valid(event)
-      invalid = []
-      invalid.push('api host') if event.api_host.nil? || event.api_host.empty?
-      invalid.push('write key') if event.writekey.nil? || event.writekey.empty?
-      invalid.push('dataset') if event.dataset.nil? || event.dataset.empty?
-
-      unless invalid.empty?
-        e = StandardError.new("#{self.class.name}: nil or empty required fields (#{invalid.join(', ')})"\
-          '. Will not attempt to send.')
-        Response.new(error: e).tap do |error_response|
-          error_response.metadata = event.metadata
-          enqueue_response(error_response)
-        end
-
-        return false
-      end
-
-      true
-    end
-
     def send_loop
       http_clients = build_http_clients
 
@@ -133,23 +113,28 @@ module Libhoney
     end
 
     def close(drain)
-      # if drain is false, clear the remaining unprocessed events from the queue
-      unless drain
-        @batch_queue.clear
-        @send_queue.clear
-      end
+      @lock.synchronize do
+        # if drain is false, clear the remaining unprocessed events from the queue
+        if drain
+          warn "#{self.class.name} - close: draining events" if %w[debug trace].include?(ENV['LOG_LEVEL'])
+        else
+          warn "#{self.class.name} - close: deleting unsent events" if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          @batch_queue.clear
+          @send_queue.clear
+        end
 
-      @batch_queue.enq(nil)
-      @batch_thread.join unless @batch_thread.nil?
+        @batch_queue.enq(nil)
+        @batch_thread&.join(1.0) # limit the amount of time we'll wait for the thread to end
 
-      # send @threads.length number of nils so each thread will fall out of send_loop
-      @threads.length.times { @send_queue << nil }
+        # send @threads.length number of nils so each thread will fall out of send_loop
+        @threads.length.times { @send_queue << nil }
 
-      @threads.each(&:join)
-      @threads = []
+        @threads.each(&:join)
+        @threads = []
+      end
 
       enqueue_response(nil)
-
+      warn "#{self.class.name} - close: close complete" if %w[debug trace].include?(ENV['LOG_LEVEL'])
       0
     end
 
@@ -161,25 +146,79 @@ module Libhoney
 
       loop do
         begin
+          # a timeout expiration waiting for an event
+          #   1. interrupts only when thread is in a blocking state (waiting for pop)
+          #   2. exception skips the break and is rescued
+          #   3. triggers the ensure to flush the current batch
+          #   3. begins the loop again with an updated next_send_time
           Thread.handle_interrupt(Timeout::Error => :on_blocking) do
+            # an event on the batch_queue
+            #   1. pops out and is truthy
+            #   2. gets included in the current batch
+            #   3. while waits for another event
             while (event = Timeout.timeout(@send_frequency) { @batch_queue.pop })
               key = [event.api_host, event.writekey, event.dataset]
               batched_events[key] << event
             end
           end
 
+          # a nil on the batch_queue
+          #   1. pops out and is falsy
+          #   2. ends the event-popping while do..end
+          #   3. breaks the loop
+          #   4. flushes the current batch
+          #   5. ends the batch_loop
           break
-        rescue Exception
+        rescue Timeout::Error
+          # Timeout::Error happens when there is nothing to pop from the batch_queue.
+          # We rescue it here to avoid spamming the logs with "execution expired" errors.
+        rescue Exception => e
+          warn "#{self.class.name}: 💥 " + e.message if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          warn e.backtrace.join("\n").to_s if ['trace'].include?(ENV['LOG_LEVEL'])
+
+        # regardless of the exception, figure out whether enough time has passed to
+        # send the current batched events, if so, send them and figure out the next send time
+        # before going back to the top of the loop
         ensure
           next_send_time = flush_batched_events(batched_events) if Time.now > next_send_time
         end
       end
 
+      # don't need to capture the next_send_time here because the batch_loop is exiting
+      # for some reason (probably transmission.close)
       flush_batched_events(batched_events)
     end
 
     private
 
+    def setup_batch_queue
+      # use a SizedQueue so the producer will block on adding to the batch_queue when @block_on_send is true
+      @batch_queue = SizedQueue.new(@pending_work_capacity)
+    end
+
+    REQUIRED_EVENT_FIELDS = %i[api_host writekey dataset].freeze
+
+    def event_valid(event)
+      missing_required_fields = REQUIRED_EVENT_FIELDS.select do |required_field|
+        event.public_send(required_field).nil? || event.public_send(required_field).empty?
+      end
+
+      if missing_required_fields.empty?
+        true
+      else
+        enqueue_response(
+          Response.new(
+            metadata: event.metadata,
+            error: StandardError.new(
+              "#{self.class.name}: nil or empty required fields (#{missing_required_fields.join(', ')})"\
+              '. Will not attempt to send.'
+            )
+          )
+        )
+        false
+      end
+    end
+
     ##
     # Enqueues a response to the responses queue suppressing ThreadError when
     # there is no space left on the queue and we are not blocking on response
@@ -190,15 +229,36 @@ module Libhoney
     end
 
     def process_response(http_response, before, batch)
-      index = 0
-      JSON.parse(http_response.body).each do |event|
-        index += 1 while batch[index].nil? && index < batch.size
-        break unless (batched_event = batch[index])
-
-        Response.new(status_code: event['status']).tap do |response|
-          response.duration = Time.now - before
-          response.metadata = batched_event.metadata
-          enqueue_response(response)
+      if http_response.status == 200
+        index = 0
+        JSON.parse(http_response.body).each do |event|
+          index += 1 while batch[index].nil? && index < batch.size
+          break unless (batched_event = batch[index])
+
+          enqueue_response(
+            Response.new(
+              status_code: event['status'],
+              duration: (Time.now - before),
+              metadata: batched_event.metadata
+            )
+          )
+        end
+      else
+        error = JSON.parse(http_response.body)['error']
+        if %w[debug trace].include?(ENV['LOG_LEVEL'])
+          warn "#{self.class.name}: error sending data to Honeycomb - #{http_response.status} #{error}"
+        end
+        batch.each do |batched_event|
+          next unless batched_event # skip nils enqueued from serialization errors
+
+          enqueue_response(
+            Response.new(
+              status_code: http_response.status, # single error from API applied to all events sent in batch
+              duration: (Time.now - before),
+              metadata: batched_event.metadata,
+              error: RuntimeError.new(error)
+            )
+          )
         end
       end
     end
@@ -234,14 +294,15 @@ module Libhoney
     end
 
     def build_user_agent(user_agent_addition)
-      ua = "libhoney-rb/#{VERSION}"
-      ua << " #{user_agent_addition}" if user_agent_addition
-      ua
+      "libhoney-rb/#{VERSION}"
+        .concat(" #{user_agent_addition}")
+        .strip # remove trailing spaces if addition was empty
+        .concat(" Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM})")
     end
 
     def ensure_threads_running
       @lock.synchronize do
-        @batch_thread = Thread.new { batch_loop } unless @batch_thread && @batch_thread.alive?
+        @batch_thread = Thread.new { batch_loop } unless @batch_thread&.alive?
         @threads.select!(&:alive?)
         @threads << Thread.new { send_loop } while @threads.length < @max_concurrent_batches
       end

data/lib/libhoney/version.rb

@@ -1,3 +1,3 @@
 module Libhoney
-  VERSION = '1.18.0'.freeze
+  VERSION = '2.0.0'.freeze
 end

data/libhoney.gemspec

@@ -20,20 +20,22 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.2.0'
+  spec.required_ruby_version = '>= 2.4.0'
 
   spec.add_development_dependency 'bump', '~> 0.5'
   spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'lockstep'
   spec.add_development_dependency 'minitest', '~> 5.0'
-  spec.add_development_dependency 'rake', '~> 12.3'
-  spec.add_development_dependency 'rubocop', '< 0.69'
+  spec.add_development_dependency 'minitest-reporters'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rubocop', '1.12.1'
   spec.add_development_dependency 'sinatra'
   spec.add_development_dependency 'sinatra-contrib'
-  spec.add_development_dependency 'spy', '1.0.0'
+  spec.add_development_dependency 'spy', '~> 1.0'
   spec.add_development_dependency 'webmock', '~> 3.4'
   spec.add_development_dependency 'yard'
   spec.add_development_dependency 'yardstick', '~> 0.9'
   spec.add_dependency 'addressable', '~> 2.0'
   spec.add_dependency 'excon'
-  spec.add_dependency 'http', '>= 2.0', '< 5.0'
+  spec.add_dependency 'http', '>= 2.0', '< 6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libhoney
 version: !ruby/object:Gem::Version
-  version: 1.18.0
+  version: 2.0.0
 platform: ruby
 authors:
 - The Honeycomb.io Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-14 00:00:00.000000000 Z
+date: 2021-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: lockstep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -52,34 +66,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.69'
+        version: 1.12.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.69'
+        version: 1.12.1
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
@@ -112,16 +140,16 @@ dependencies:
   name: spy
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -201,7 +229,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -211,7 +239,7 @@ dependencies:
         version: '2.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 description: Ruby gem for sending data to Honeycomb
 email: support@honeycomb.io
 executables: []
@@ -222,27 +250,46 @@ files:
 - ".circleci/setup-rubygems.sh"
 - ".editorconfig"
 - ".github/CODEOWNERS"
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/ISSUE_TEMPLATE/question-discussion.md"
+- ".github/ISSUE_TEMPLATE/security-vulnerability-report.md"
+- ".github/PULL_REQUEST_TEMPLATE.md"
+- ".github/dependabot.yml"
+- ".github/workflows/add-to-project.yml"
+- ".github/workflows/apply-labels.yml"
+- ".github/workflows/stale.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
 - CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE
+- OSSMETADATA
 - README.md
+- RELEASING.md
 - Rakefile
+- SECURITY.md
+- SUPPORT.md
 - example/factorial.rb
 - lib/libhoney.rb
 - lib/libhoney/builder.rb
 - lib/libhoney/cleaner.rb
 - lib/libhoney/client.rb
 - lib/libhoney/event.rb
+- lib/libhoney/experimental_transmission.rb
 - lib/libhoney/log_client.rb
 - lib/libhoney/log_transmission.rb
 - lib/libhoney/mock_transmission.rb
 - lib/libhoney/null_client.rb
 - lib/libhoney/null_transmission.rb
+- lib/libhoney/queueing.rb
+- lib/libhoney/queueing/LICENSE.txt
+- lib/libhoney/queueing/sized_queue_with_timeout.rb
 - lib/libhoney/response.rb
 - lib/libhoney/test_client.rb
 - lib/libhoney/transmission.rb
@@ -260,14 +307,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: send data to Honeycomb