brow 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffdc8b811ca1be5ce149f0e21dc0c08a6f8a4f2e79368f20a69503ca33e2d997
-  data.tar.gz: 4f58ddc175db9c12b6fa77b13fe9f1d82a49100ea1d1cc143d335cb9a5db082b
+  metadata.gz: 94b1026b806df0a58a01aa0bc911ca756251dc5d0de510ae1c12d42665b00559
+  data.tar.gz: 55b2b5454086ac78e22681ced81bb6d500dd9c6537ebfcf67bbc9a445a2d9a79
 SHA512:
-  metadata.gz: e66c43a44fd5e10b3aba344941bbc046ed1943208da8de20bc2a96577d1642d94e3528586a1db3d7ee998c45e966bc8daa72de39933a8dfa7354a0f737c1e4b1
-  data.tar.gz: 704c98568783c393b58f438d23039003c816b7c363932785ceb2f747ddfca6e7da049708c7fd7e6c22f10e392b65928367bbd0d5667aea9720e7a7f7259747c2
+  metadata.gz: 2d23c1ff291ca611a65b1357392e2f4275502fc3467a629e85ff631f7585050095ef4cc86f0b86981fb0e7e790ce709bd2a0e2b511f0e706954cec6221ae40db
+  data.tar.gz: 884ffd8cebbcaf2681a1347a1115892676c86de12dcef693e5364531e37cef0d3700343cb8e4d2393270003c541d9efbd0e1c427fef904b66f475eb81ece2246

data/CHANGELOG.md

@@ -1,9 +1,25 @@
 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0] - 2021-10-29
+
+https://github.com/jnunemaker/brow/pull/4
+
+### Fixed
+
+- Fixed thread churn. Upon digging in, I realized that the previous code was creating a bunch of threads. Basically one for each batch, which seems far from ideal. I'm surprised it worked that way. This changes it to be one worker thread that just sits there forever in a loop. When a batch is full, it transports it. When shutdown happens, a shutdown message is enqueued and the worker breaks the loop.
+- Moved worker thread management to `Worker` from `Client`.
+- Back off policy is now reset after `Transport#send_batch` completes. Previously it wasn't, which meant the next interval would get to the max and stay there.
+
+### Changed
+
+- Switched to stringify data keys instead of symbolize. Old versions of ruby didn't gc symbols so that was a memory leak. Might be fixed now, but strings are fine here so lets roll with them.
+- Removed test mode and test queue. I didn't like this implementation and neither did @bkeepers. We'll come up with something new and better soon like Brow::Clients::Memory.new or something.
+
 ## [0.2.0] - 2021-10-25
 
 ### Changed

data/Gemfile

@@ -5,6 +5,7 @@ gemspec
 
 gem "rake", "~> 13.0"
 gem "minitest", "~> 5.0"
+gem "maxitest", "~> 4.1"
 gem "minitest-heat", "~> 0.0"
 gem "webmock", "~> 3.10.0"
 gem "rack", "~> 2.2.3"

data/examples/basic.rb

@@ -1,12 +1,20 @@
 require_relative "../lib/brow"
 
 client = Brow::Client.new({
-  url: "https://requestbin.net/r/4f09194m",
+  url: "https://requestbin.net/r/2bp3p3vn",
+  batch_size: 10,
 })
 
-150.times do |n|
-  client.push({
-    number: n,
-    now: Time.now.utc,
-  })
+5.times do |n|
+  client.push(n: n, parent: true)
 end
+
+pid = fork {
+  15.times do |n|
+    client.push({
+      number: n,
+      now: Time.now.utc,
+    })
+  end
+}
+Process.waitpid pid, 0

data/lib/brow/backoff_policy.rb

@@ -6,7 +6,7 @@ module Brow
     MIN_TIMEOUT_MS = 100
 
     # Private: The default maximum timeout between intervals in milliseconds.
-    MAX_TIMEOUT_MS = 10000
+    MAX_TIMEOUT_MS = 10_000
 
     # Private: The value to multiply the current interval with for each
     # retry attempt.
@@ -16,6 +16,12 @@ module Brow
     # retry interval.
     RANDOMIZATION_FACTOR = 0.5
 
+    # Private
+    attr_reader :min_timeout_ms, :max_timeout_ms, :multiplier, :randomization_factor
+
+    # Private
+    attr_reader :attempts
+
     # Public: Create new instance of backoff policy.
     #
     # options - The Hash of options.
@@ -44,6 +50,10 @@ module Brow
       [interval, @max_timeout_ms].min
     end
 
+    def reset
+      @attempts = 0
+    end
+
     private
 
     def add_jitter(base, randomization_factor)

data/lib/brow/client.rb

@@ -1,139 +1,69 @@
 # frozen_string_literal: true
 
-require 'thread'
 require 'time'
 
 require_relative 'utils'
 require_relative 'worker'
-require_relative 'test_queue'
 
 module Brow
   class Client
-    # Private: Default # of items that can be in queue before we start dropping data.
-    MAX_QUEUE_SIZE = 10_000
-
-    # Private: Default number of seconds to wait to shutdown worker thread.
-    SHUTDOWN_TIMEOUT = 5
-
     # Public: Create a new instance of a client.
     #
     # options - The Hash of options.
+    #   :url - The URL where all batches of data should be transported.
     #   :max_queue_size - The maximum number of calls to be remain queued.
+    #   :logger - The Logger to use to log useful information about what is
+    #             going on.
+    #   :queue - The Queue to use to store data until it can be batched up and
+    #            transported to the API.
+    #   :worker - The Worker that will pop items off the queue, batch them up
+    #             and transport them to the API.
+    #   :transport - The Transport to use to transport batches to the API.
+    #   :headers - The Hash of headers to include when transporting batches to
+    #              the API. These could be used for auth or whatever.
+    #   :retries - The Integer number of times the transport should retry a call
+    #              before giving up.
+    #   :read_timeout - The number of seconds to wait when reading data before
+    #                   giving up.
+    #   :open_timeout - The number of seconds to wait when opening a connection
+    #                   to the API.
+    #   :backoff_policy - The BackoffPolicy to use to determine when the next
+    #                     retry should occur when the transport fails to send a
+    #                     batch of data to the API.
+    #   :min_timeout_ms - The minimum number of milliseconds to wait before
+    #                     retrying a failed call to the API.
+    #   :max_timeout_ms - The maximum number of milliseconds to wait before
+    #                     retrying a failed call to the API.
+    #   :multiplier - The value to multily the current interval with for each
+    #                 retry attempt.
+    #   :randomization_factor - The value to use to create a range of jitter
+    #                 around the retry interval.
+    #   :batch - The MessageBatch used to batch up several events to be
+    #            transported in one call to the API.
+    #   :shutdown_timeout - The number of seconds to wait for the worker thread
+    #                       to join when shutting down.
+    #   :shutdown_automatically - Should the worker shutdown automatically or
+    #                             manually. If true, shutdown is automatic. If
+    #                             false, you'll need to handle this on your own.
+    #   :max_size - The maximum number of items a batch can contain before it
+    #               should be transported to the API. Only used if not :batch
+    #               is provided.
     #   :on_error - The Proc that handles error calls from the API.
     def initialize(options = {})
       options = Brow::Utils.symbolize_keys(options)
-
-      @worker_thread = nil
-      @worker_mutex = Mutex.new
-      @pid = Process.pid
-      @test = options[:test]
-      @max_queue_size = options[:max_queue_size] || MAX_QUEUE_SIZE
-      @logger = options.fetch(:logger) { Brow.logger }
-      @queue = options.fetch(:queue) { Queue.new }
-      @worker = options.fetch(:worker) { Worker.new(@queue, options) }
-      @shutdown_timeout = options.fetch(:shutdown_timeout) { SHUTDOWN_TIMEOUT }
-
-      if options.fetch(:shutdown_automatically, true)
-        at_exit { shutdown }
-      end
+      @worker = options.fetch(:worker) { Worker.new(options) }
     end
 
-    # Public: Synchronously waits until the worker has flushed the queue.
-    #
-    # Use only for scripts which are not long-running, and will
-    # specifically exit.
-    def flush
-      while !@queue.empty? || @worker.requesting?
-        ensure_threads_alive
-        sleep(0.1)
-      end
-    end
-
-    def shutdown
-      if @worker_thread
-        begin
-          @worker_thread.join @shutdown_timeout
-        rescue => error
-          @logger.info("[brow]") { "Error shutting down worker thread: #{error.inspect}"}
-        end
-      end
-    end
+    # Private
+    attr_reader :worker
 
     # Public: Enqueues an event to eventually be transported to backend service.
     #
-    # event - The Hash of event data.
+    # data - The Hash of data.
     #
-    # Returns Boolean of whether the item was added to the queue.
-    def push(item)
-      raise ArgumentError, "item must be a Hash" unless item.is_a?(Hash)
-
-      item = Brow::Utils.symbolize_keys(item)
-      item = Brow::Utils.isoify_dates(item)
-
-      if @test
-        test_queue << item
-        return true
-      end
-
-      ensure_threads_alive
-
-      if @queue.length < @max_queue_size
-        @queue << item
-        true
-      else
-        @logger.warn("[brow]") { "Queue is full, dropping events. The :max_queue_size configuration parameter can be increased to prevent this from happening." }
-        false
-      end
-    end
-
-    # Public: Returns the number of messages in the queue.
-    def queued_messages
-      @queue.length
-    end
-
-    # Public: For test purposes only. If test: true is passed to #initialize
-    # then all pushing of events will go to test queue in memory so they can
-    # be verified with assertions.
-    def test_queue
-      unless @test
-        raise 'Test queue only available when setting :test to true.'
-      end
-
-      @test_queue ||= TestQueue.new
-    end
-
-    private
-
-    def forked?
-      @pid != Process.pid
-    end
-
-    def ensure_threads_alive
-      reset if forked?
-      ensure_worker_running
-    end
-
-    def ensure_worker_running
-      # If another thread is starting worker thread, then return early so this
-      # thread can enqueue and move on with life.
-      return unless @worker_mutex.try_lock
-
-      begin
-        return if worker_running?
-        @worker_thread = Thread.new { @worker.run }
-      ensure
-        @worker_mutex.unlock
-      end
-    end
-
-    def reset
-      @pid = Process.pid
-      @worker_mutex.unlock if @worker_mutex.locked?
-      @queue.clear
-    end
-
-    def worker_running?
-      @worker_thread && @worker_thread.alive?
+    # Returns Boolean of whether the data was added to the queue.
+    def push(data)
+      worker.push(data)
     end
   end
 end

data/lib/brow/message_batch.rb

@@ -22,6 +22,8 @@ module Brow
 
     def_delegators :@messages, :empty?
     def_delegators :@messages, :length
+    def_delegators :@messages, :size
+    def_delegators :@messages, :count
 
     attr_reader :uuid, :json_size
 

data/lib/brow/transport.rb

@@ -9,21 +9,17 @@ require_relative 'backoff_policy'
 
 module Brow
   class Transport
+    # Private: Default number of times to retry request.
     RETRIES = 10
+
+    # Private: Default read timeout on requests.
     READ_TIMEOUT = 8
+
+    # Private: Default open timeout on requests.
     OPEN_TIMEOUT = 4
-    HEADERS = {
-      "Accept" => "application/json",
-      "Content-Type" => "application/json",
-      "User-Agent" => "brow-ruby/#{Brow::VERSION}",
-      "Client-Language" => "ruby",
-      "Client-Language-Version" => "#{RUBY_VERSION} p#{RUBY_PATCHLEVEL} (#{RUBY_RELEASE_DATE})",
-      "Client-Platform" => RUBY_PLATFORM,
-      "Client-Engine" => defined?(RUBY_ENGINE) ? RUBY_ENGINE : "",
-      "Client-Hostname" => Socket.gethostname,
-    }
-
-    attr_reader :url
+
+    # Private
+    attr_reader :url, :headers, :retries, :logger, :backoff_policy, :http
 
     def initialize(options = {})
       @url = options[:url] || raise(ArgumentError, ":url is required to be present so we know where to send batches")
@@ -34,7 +30,7 @@ module Brow
         @uri.path = "/"
       end
 
-      @headers = HEADERS.merge(options[:headers] || {})
+      @headers = options[:headers] || {}
       @retries = options[:retries] || RETRIES
 
       @logger = options.fetch(:logger) { Brow.logger }
@@ -52,25 +48,29 @@ module Brow
     #
     # @return [Response] API response
     def send_batch(batch)
-      @logger.debug("[brow]") { "Sending request for #{batch.length} items" }
+      logger.debug("[brow]") { "Sending request for #{batch.length} items" }
 
-      last_response, exception = retry_with_backoff(@retries) do
+      last_response, exception = retry_with_backoff(retries) do
         response = send_request(batch)
-        @logger.debug("[brow]") { "Response: status=#{response.code}, body=#{response.body}" }
+        logger.debug("[brow]") { "Response: status=#{response.code}, body=#{response.body}" }
         [Response.new(response.code.to_i, nil), retry?(response)]
       end
 
       if exception
-        @logger.error("[brow]") { exception.message }
-        exception.backtrace.each { |line| @logger.error(line) }
+        logger.error("[brow]") { exception.message }
+        exception.backtrace.each { |line| logger.error(line) }
         Response.new(-1, exception.to_s)
       else
         last_response
       end
+    ensure
+      backoff_policy.reset
+      batch.clear
     end
 
     # Closes a persistent connection if it exists
     def shutdown
+      logger.info("[brow]") { "Transport shutting down" }
       @http.finish if @http.started?
     end
 
@@ -80,15 +80,15 @@ module Brow
       status_code = response.code.to_i
       if status_code >= 500
         # Server error. Retry and log.
-        @logger.info("[brow]") { "Server error: status=#{status_code}, body=#{response.body}" }
+        logger.info("[brow]") { "Server error: status=#{status_code}, body=#{response.body}" }
         true
       elsif status_code == 429
         # Rate limited. Retry and log.
-        @logger.info("[brow]") { "Rate limit error: body=#{response.body}" }
+        logger.info("[brow]") { "Rate limit error: body=#{response.body}" }
         true
       elsif status_code >= 400
         # Client error. Do not retry, but log.
-        @logger.error("[brow]") { "Client error: status=#{status_code}, body=#{response.body}" }
+        logger.error("[brow]") { "Client error: status=#{status_code}, body=#{response.body}" }
         false
       else
         false
@@ -110,13 +110,13 @@ module Brow
         result, should_retry = yield
         return [result, nil] unless should_retry
       rescue StandardError => error
-        @logger.debug("[brow]") {  "Request error: #{error}" }
+        logger.debug("[brow]") {  "Request error: #{error}" }
         should_retry = true
         caught_exception = error
       end
 
       if should_retry && (retries_remaining > 1)
-        @logger.debug("[brow]") { "Retrying request, #{retries_remaining} retries left" }
+        logger.debug("[brow]") { "Retrying request, #{retries_remaining} retries left" }
         sleep(@backoff_policy.next_interval.to_f / 1000)
         retry_with_backoff(retries_remaining - 1, &block)
       else
@@ -126,6 +126,14 @@ module Brow
 
     def send_request(batch)
       headers = {
+        "Accept" => "application/json",
+        "Content-Type" => "application/json",
+        "User-Agent" => "brow-ruby/#{Brow::VERSION}",
+        "Client-Language" => "ruby",
+        "Client-Language-Version" => "#{RUBY_VERSION} p#{RUBY_PATCHLEVEL} (#{RUBY_RELEASE_DATE})",
+        "Client-Platform" => RUBY_PLATFORM,
+        "Client-Engine" => defined?(RUBY_ENGINE) ? RUBY_ENGINE : "",
+        "Client-Hostname" => Socket.gethostname,
         "Client-Pid" => Process.pid.to_s,
         "Client-Thread" => Thread.current.object_id.to_s,
       }.merge(@headers)

data/lib/brow/utils.rb

@@ -13,6 +13,13 @@ module Brow
       end
     end
 
+    # Internal: Return a new hash with keys converted to strings
+    def stringify_keys(hash)
+      hash.each_with_object({}) do |(k, v), memo|
+        memo[k.to_s] = v
+      end
+    end
+
     # Internal: Returns a new hash with all the date values in the into
     # iso8601 strings
     def isoify_dates(hash)

data/lib/brow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Brow
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/brow/worker.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'thread'
+
 require_relative 'message_batch'
 require_relative 'transport'
 require_relative 'utils'
@@ -7,8 +9,24 @@ require_relative 'utils'
 module Brow
   # Internal: The Worker to pull items off the queue and put them
   class Worker
+    # Private: Noop default on error proc.
     DEFAULT_ON_ERROR = proc { |response| }
 
+    # Private: Object to enqueue to signal shutdown for worker.
+    SHUTDOWN = :__ಠ_ಠ__
+
+    # Private: Default number of seconds to wait to shutdown worker thread.
+    SHUTDOWN_TIMEOUT = 5
+
+    # Private: Default # of items that can be in queue before we start dropping data.
+    MAX_QUEUE_SIZE = 10_000
+
+    # Private
+    attr_reader :thread, :queue, :pid, :mutex, :on_error, :batch_size, :max_queue_size
+
+    # Private
+    attr_reader :logger, :transport, :shutdown_timeout
+
     # Internal: Creates a new worker
     #
     # The worker continuously takes messages off the queue and makes requests to
@@ -16,50 +34,130 @@ module Brow
     #
     # queue   - Queue synchronized between client and worker
     # options - The Hash of worker options.
-    #           batch_size - Fixnum of how many items to send in a batch.
-    #           on_error - Proc of what to do on an error.
-    #           transport - The Transport object to deliver batches.
-    #           logger - The Logger object for all log messages.
-    #           batch - The MessageBatch to collect messages and deliver batches
-    #                   via Transport.
-    def initialize(queue, options = {})
-      @queue = queue
-      @lock = Mutex.new
+    #   :on_error - Proc of what to do on an error.
+    #   :batch_size - Fixnum of how many items to send in a batch.
+    #   :transport - The Transport object to deliver batches.
+    #   :logger - The Logger object for all log messages.
+    #   :batch - The MessageBatch to collect messages and deliver batches
+    #            via Transport.
+    #   :shutdown_timeout - The number of seconds to wait for the worker thread
+    #                       to join when shutting down.
+    #   :shutdown_automatically - Should the client shutdown automatically or
+    #                             manually. If true, shutdown is automatic. If
+    #                             false, you'll need to handle this on your own.
+    def initialize(options = {})
+      @thread = nil
+      @queue = options.fetch(:queue) { Queue.new }
+      @pid = Process.pid
+      @mutex = Mutex.new
       options = Brow::Utils.symbolize_keys(options)
       @on_error = options[:on_error] || DEFAULT_ON_ERROR
+      @batch_size = options[:batch_size]
+      @max_queue_size = options.fetch(:max_queue_size) { MAX_QUEUE_SIZE }
+      @logger = options.fetch(:logger) { Brow.logger }
       @transport = options.fetch(:transport) { Transport.new(options) }
-      @batch = options.fetch(:batch) { MessageBatch.new(max_size: options[:batch_size]) }
+      @shutdown_timeout = options.fetch(:shutdown_timeout) { SHUTDOWN_TIMEOUT }
+
+      if options.fetch(:shutdown_automatically, true)
+        at_exit { stop }
+      end
+    end
+
+    def push(data)
+      raise ArgumentError, "data must be a Hash" unless data.is_a?(Hash)
+      start
+
+      data = Utils.isoify_dates(Utils.stringify_keys(data))
+
+      if queue.length < max_queue_size
+        queue << data
+        true
+      else
+        logger.warn("[brow]") { "Queue is full, dropping events. The :max_queue_size configuration parameter can be increased to prevent this from happening." }
+        false
+      end
+    end
+
+    def start
+      reset if forked?
+      ensure_worker_running
+    end
+
+    def stop
+      queue << SHUTDOWN
+
+      if @thread
+        begin
+          if @thread.join(shutdown_timeout)
+            logger.info("[brow]") { "Worker thread [#{@thread.object_id}] joined sucessfully" }
+          else
+            logger.info("[brow]") { "Worker thread [#{@thread.object_id}] did not join successfully" }
+          end
+        rescue => error
+          logger.info("[brow]") { "Worker thread [#{@thread.object_id}] error shutting down: #{error.inspect}" }
+        end
+      end
     end
 
     # Internal: Continuously runs the loop to check for new events
     def run
-      until Thread.current[:should_exit]
-        return if @queue.empty?
+      batch = MessageBatch.new(max_size: batch_size)
 
-        @lock.synchronize do
-          consume_message_from_queue! until @batch.full? || @queue.empty?
-        end
+      loop do
+        message = queue.pop
 
-        response = @transport.send_batch @batch
-        @on_error.call(response) unless response.status == 200
+        case message
+        when SHUTDOWN
+          send_batch(batch) unless batch.empty?
+          break
+        else
+          begin
+            batch << message
+          rescue MessageBatch::JSONGenerationError => error
+            on_error.call(Response.new(-1, error))
+          end
 
-        @lock.synchronize { @batch.clear }
+          send_batch(batch) if batch.full?
+        end
       end
     ensure
-      @transport.shutdown
+      transport.shutdown
     end
 
-    # Internal: Check whether we have outstanding requests.
-    def requesting?
-      @lock.synchronize { !@batch.empty? }
+    private
+
+    def forked?
+      pid != Process.pid
     end
 
-    private
+    def ensure_worker_running
+      # If another thread is starting worker thread, then return early so this
+      # thread can enqueue and move on with life.
+      return unless mutex.try_lock
+
+      begin
+        return if @thread && @thread.alive?
+        @thread = Thread.new { run }
+        logger.debug("[brow]") { "Worker thread [#{@thread.object_id}] started" }
+      ensure
+        mutex.unlock
+      end
+    end
+
+    def reset
+      @pid = Process.pid
+      mutex.unlock if mutex.locked?
+      queue.clear
+    end
+
+    def send_batch(batch)
+      response = transport.send_batch(batch)
+
+      unless response.status == 200
+        on_error.call(response)
+      end
 
-    def consume_message_from_queue!
-      @batch << @queue.pop
-    rescue MessageBatch::JSONGenerationError => error
-      @on_error.call(Response.new(-1, error))
+      response
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: brow
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - John Nunemaker
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-26 00:00:00.000000000 Z
+date: 2021-10-29 00:00:00.000000000 Z
 dependencies: []
 description: 
 email:
@@ -31,7 +31,6 @@ files:
 - lib/brow/client.rb
 - lib/brow/message_batch.rb
 - lib/brow/response.rb
-- lib/brow/test_queue.rb
 - lib/brow/transport.rb
 - lib/brow/utils.rb
 - lib/brow/version.rb

data/lib/brow/test_queue.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Brow
-  # Public: The test queue to use if the `Client` is in test mode. Keeps all
-  # messages in an array so you can add assertions.
-  #
-  # Be sure to reset before each test case.
-  class TestQueue
-    attr_reader :messages
-
-    def initialize
-      reset
-    end
-
-    def count
-      messages.count
-    end
-    alias_method :size, :count
-    alias_method :length, :count
-
-    def <<(message)
-      messages << message
-    end
-
-    def reset
-      @messages = []
-    end
-  end
-end