libhoney 1.12.1 → 1.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6c5ff396b0f0a043e1ee4b1ae1d89342fdfbc99861e5cdb22d7c5843acd112b
-  data.tar.gz: ee861538fa2020c201dc5be4477372c9778c89b910d0f8e2faa2168371ea6f97
+  metadata.gz: '085a461457305c6c626057ffa084b66bb3d364d3bf538a66116de45c8195b78e'
+  data.tar.gz: 25cdef56d6c38db3b91f411763049acf5f536514002c1ca4ad519c406cb4309c
 SHA512:
-  metadata.gz: 3e53c57343b02b2de9b028a06ed6800a78593267169ed9ea302cc2a7783bdc2ea4db4053321bd97162b0afdf9d8a97e3abbcd3c63f599d5c2ef02b69ea7fe886
-  data.tar.gz: 4d513bbdd0decf922d3a6462a9dd644805cec48d353a2059286b5e408fb3ff94554300fd0ffe1b7028b21de6fbe2970e5a0effa7b1f4647f3fe812db00ab1e2c
+  metadata.gz: 5bd46c22d978845d9b124fda903632186ab69003f8a319db662718e222da0d18ce2215a9f54c71e413856a112b32fb6871a5a9d5f1539121120da2ea5476b30e
+  data.tar.gz: 8982666d9a862224d7ce2200ac79f0876ded8ecc05a6b010c7ff57e77cb90e978c8ffb64fa1edbc3b1951a2b0ab4c1ba2722c9bacc1a24ee980347535f7caf93

data/.rubocop.yml

@@ -10,7 +10,11 @@ Lint/RescueException:
   Exclude:
     - 'lib/libhoney/transmission.rb'
 
+Metrics/BlockLength:
+  Max: 35
+
 Metrics/ClassLength:
+  Max: 200
   Exclude:
     - test/*
 

data/lib/libhoney/response.rb

@@ -5,11 +5,11 @@ module Libhoney
     attr_accessor :duration, :status_code, :metadata, :error
 
     def initialize(duration: 0,
-                   status_code: HTTP::Response::Status.new(0),
+                   status_code: 0,
                    metadata: nil,
                    error: nil)
       @duration    = duration
-      @status_code = status_code
+      @status_code = HTTP::Response::Status.new(status_code)
       @metadata    = metadata
       @error       = error
     end

data/lib/libhoney/transmission.rb

@@ -1,3 +1,5 @@
+require 'json'
+require 'timeout'
 require 'libhoney/response'
 
 module Libhoney
@@ -17,16 +19,19 @@ module Libhoney
       @block_on_send          = block_on_send
       @block_on_responses     = block_on_responses
       @max_batch_size         = max_batch_size
-      @send_frequency         = send_frequency
+      # convert to seconds
+      @send_frequency         = send_frequency.fdiv(1000)
       @max_concurrent_batches = max_concurrent_batches
       @pending_work_capacity  = pending_work_capacity
       @send_timeout           = send_timeout
       @user_agent             = build_user_agent(user_agent_addition).freeze
 
-      # use a SizedQueue so the producer will block on adding to the send_queue when @block_on_send is true
-      @send_queue = SizedQueue.new(@pending_work_capacity)
-      @threads    = []
-      @lock       = Mutex.new
+      @send_queue   = Queue.new
+      @threads      = []
+      @lock         = Mutex.new
+      # use a SizedQueue so the producer will block on adding to the batch_queue when @block_on_send is true
+      @batch_queue  = SizedQueue.new(@pending_work_capacity)
+      @batch_thread = nil
     end
 
     def add(event)
@@ -35,7 +40,7 @@ module Libhoney
       raise ArgumentError, "No Dataset for Honeycomb. Can't send datasetless."          if event.dataset  == ''
 
       begin
-        @send_queue.enq(event, !@block_on_send)
+        @batch_queue.enq(event, !@block_on_send)
       rescue ThreadError
         # happens if the queue was full and block_on_send = false.
       end
@@ -44,57 +49,53 @@ module Libhoney
     end
 
     def send_loop
-      http_clients = Hash.new do |h, api_host|
-        h[api_host] = HTTP.timeout(connect: @send_timeout, write: @send_timeout, read: @send_timeout)
-                          .persistent(api_host)
-                          .headers(
-                            'User-Agent' => @user_agent,
-                            'Content-Type' => 'application/json'
-                          )
-      end
+      http_clients = build_http_clients
 
       # eat events until we run out
       loop do
-        event = @send_queue.pop
-        break if event.nil?
+        api_host, writekey, dataset, batch = @send_queue.pop
+        break if batch.nil?
 
         before = Time.now
 
         begin
-          http = http_clients[event.api_host]
-          url  = '/1/events/' + Addressable::URI.escape(event.dataset.dup)
-
-          resp = http.post(url,
-                           json: event.data,
-                           headers: {
-                             'X-Honeycomb-Team' => event.writekey,
-                             'X-Honeycomb-SampleRate' => event.sample_rate,
-                             'X-Event-Time' => event.timestamp.iso8601(3)
-                           })
-
-          # "You must consume response before sending next request via persistent connection"
-          # https://github.com/httprb/http/wiki/Persistent-Connections-%28keep-alive%29#note-using-persistent-requests-correctly
-          resp.flush
-
-          response = Response.new(status_code: resp.status)
-        rescue Exception => error
+          http = http_clients[api_host]
+          body = serialize_batch(batch)
+          next if body.nil?
+
+          headers = {
+            'Content-Type' => 'application/json',
+            'X-Honeycomb-Team' => writekey
+          }
+
+          response = http.post(
+            "/1/batch/#{Addressable::URI.escape(dataset)}",
+            body: body,
+            headers: headers
+          )
+          process_response(response, before, batch)
+        rescue Exception => e
           # catch a broader swath of exceptions than is usually good practice,
           # because this is effectively the top-level exception handler for the
           # sender threads, and we don't want those threads to die (leaving
           # nothing consuming the queue).
-          response = Response.new(error: error)
-        ensure
-          if response
-            response.duration = Time.now - before
-            response.metadata = event.metadata
+          begin
+            batch.each do |event|
+              # nil events in the batch should already have had an error
+              # response enqueued in #serialize_batch
+              next if event.nil?
+
+              Response.new(error: e).tap do |error_response|
+                error_response.metadata = event.metadata
+                begin
+                  @responses.enq(error_response, !@block_on_responses)
+                rescue ThreadError
+                end
+              end
+            end
+          rescue ThreadError
           end
         end
-
-        begin
-          @responses.enq(response, !@block_on_responses) if response
-        rescue ThreadError
-          # happens if the queue was full and block_on_send = false.
-        end
       end
     ensure
       http_clients.each do |_, http|
@@ -108,7 +109,13 @@ module Libhoney
 
     def close(drain)
       # if drain is false, clear the remaining unprocessed events from the queue
-      @send_queue.clear if drain == false
+      unless drain
+        @batch_queue.clear
+        @send_queue.clear
+      end
+
+      @batch_queue.enq(nil)
+      @batch_thread.join
 
       # send @threads.length number of nils so each thread will fall out of send_loop
       @threads.length.times { @send_queue << nil }
@@ -121,8 +128,78 @@ module Libhoney
       0
     end
 
+    def batch_loop
+      next_send_time = Time.now + @send_frequency
+      batched_events = Hash.new do |h, key|
+        h[key] = []
+      end
+
+      loop do
+        begin
+          while (event = Timeout.timeout(@send_frequency) { @batch_queue.pop })
+            key = [event.api_host, event.writekey, event.dataset]
+            batched_events[key] << event
+          end
+
+          break
+        rescue Exception
+        ensure
+          next_send_time = flush_batched_events(batched_events) if Time.now > next_send_time
+        end
+      end
+
+      flush_batched_events(batched_events)
+    end
+
     private
 
+    def process_response(http_response, before, batch)
+      index = 0
+      http_response.parse.each do |event|
+        index += 1 while batch[index].nil? && index < batch.size
+        break unless (batched_event = batch[index])
+
+        Response.new(status_code: event['status']).tap do |response|
+          response.duration = Time.now - before
+          response.metadata = batched_event.metadata
+          begin
+            @responses.enq(response, !@block_on_responses)
+          rescue ThreadError
+          end
+        end
+      end
+    end
+
+    def serialize_batch(batch)
+      payload = []
+      batch.map! do |event|
+        begin
+          e = {
+            time: event.timestamp.iso8601(3),
+            samplerate: event.sample_rate,
+            data: event.data
+          }
+          payload << JSON.generate(e)
+
+          event
+        rescue StandardError => e
+          Response.new(error: e).tap do |response|
+            response.metadata = event.metadata
+            begin
+              @responses.enq(response, !@block_on_responses)
+            rescue ThreadError
+            end
+          end
+
+          nil
+        end
+      end
+
+      return if payload.empty?
+
+      "[#{payload.join(',')}]"
+    end
+
     def build_user_agent(user_agent_addition)
       ua = "libhoney-rb/#{VERSION}"
       ua << " #{user_agent_addition}" if user_agent_addition
@@ -131,9 +208,32 @@ module Libhoney
 
     def ensure_threads_running
       @lock.synchronize do
+        @batch_thread = Thread.new { batch_loop } unless @batch_thread && @batch_thread.alive?
         @threads.select!(&:alive?)
         @threads << Thread.new { send_loop } while @threads.length < @max_concurrent_batches
       end
     end
+
+    def flush_batched_events(batched_events)
+      batched_events.each do |(api_host, writekey, dataset), events|
+        events.each_slice(@max_batch_size) do |batch|
+          @send_queue << [api_host, writekey, dataset, batch]
+        end
+      end
+      batched_events.clear
+
+      Time.now + @send_frequency
+    end
+
+    def build_http_clients
+      Hash.new do |h, api_host|
+        h[api_host] = HTTP.timeout(connect: @send_timeout, write: @send_timeout, read: @send_timeout)
+                          .persistent(api_host)
+                          .headers(
+                            'User-Agent' => @user_agent,
+                            'Content-Type' => 'application/json'
+                          )
+      end
+    end
   end
 end

data/lib/libhoney/version.rb

@@ -1,3 +1,3 @@
 module Libhoney
-  VERSION = '1.12.1'.freeze
+  VERSION = '1.13.0'.freeze
 end

data/libhoney.gemspec

@@ -26,7 +26,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'rake', '~> 12.3'
-  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop', '< 0.69'
+  spec.add_development_dependency 'sinatra'
+  spec.add_development_dependency 'sinatra-contrib'
   spec.add_development_dependency 'webmock', '~> 3.4'
   spec.add_development_dependency 'yard'
   spec.add_development_dependency 'yardstick', '~> 0.9'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libhoney
 version: !ruby/object:Gem::Version
-  version: 1.12.1
+  version: 1.13.0
 platform: ruby
 authors:
 - The Honeycomb.io Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-13 00:00:00.000000000 Z
+date: 2019-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -68,6 +68,34 @@ dependencies:
         version: '12.3'
 - !ruby/object:Gem::Dependency
   name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.69'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.69'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra-contrib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -206,7 +234,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: send data to Honeycomb