brow 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f08f2a47a1034332966c9ef42acdecc5629052fd8255857a61cdaa8483be502
-  data.tar.gz: f42dac7bdfe22a48b7e7d5e0ef5ecb91407505f9ef0fb91393a263b194a8845c
+  metadata.gz: ffdc8b811ca1be5ce149f0e21dc0c08a6f8a4f2e79368f20a69503ca33e2d997
+  data.tar.gz: 4f58ddc175db9c12b6fa77b13fe9f1d82a49100ea1d1cc143d335cb9a5db082b
 SHA512:
-  metadata.gz: abb687ce5fe388f7c87826752255ff1227dd275365ed599d3e78519f1402a2bf4a6010e9f6da307f8f7265be8a69e6f26baea43e30b7bd9adc2cd9f3253ca447
-  data.tar.gz: 57722a879c3fa49461ffbf8334a5674333fce12bb30dc2037b7dcb09f65d3e2335e57d4abb47359a7ed2b032154e1e9494e92c3a5f1a9e00a99cd0627feb1488
+  metadata.gz: e66c43a44fd5e10b3aba344941bbc046ed1943208da8de20bc2a96577d1642d94e3528586a1db3d7ee998c45e966bc8daa72de39933a8dfa7354a0f737c1e4b1
+  data.tar.gz: 704c98568783c393b58f438d23039003c816b7c363932785ceb2f747ddfca6e7da049708c7fd7e6c22f10e392b65928367bbd0d5667aea9720e7a7f7259747c2

data/CHANGELOG.md

@@ -1,5 +1,23 @@
-## [Unreleased]
+# Changelog
+All notable changes to this project will be documented in this file.
 
-## [0.1.0] - 2021-10-14
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-- Initial release
+## [0.2.0] - 2021-10-25
+
+### Changed
+
+- [c25dce](https://github.com/jnunemaker/brow/commit/c25dcedcab2b75cfe28a561e80e537fefae6cc52) `record` is now `push`.
+
+### Fixed
+
+- [eceb02](https://github.com/jnunemaker/brow/commit/eceb02f810cc5ace7d7540c957fc1cf924849629) Fixed problems with shutdown (required a flush to get whatever batches were in progress) and forking (caused queue to not get worked off).
+
+### Added
+
+- [c7f7e4](https://github.com/jnunemaker/brow/commit/c7f7e42b0d6bfa9fa96bac58fda0ef94f93d223d) `BackoffPolicy` now gets `options` so you can pass those to `Client` and they'll make it all the way through.
+
+## [0.1.0] - 2021-10-20
+
+- Initial release. Let's face it I just wanted to squat on the gem name.

data/Gemfile

@@ -7,6 +7,7 @@ gem "rake", "~> 13.0"
 gem "minitest", "~> 5.0"
 gem "minitest-heat", "~> 0.0"
 gem "webmock", "~> 3.10.0"
+gem "rack", "~> 2.2.3"
 
 group(:guard) do
   gem "guard", "~> 2.18.0"

data/README.md

@@ -1,6 +1,6 @@
 # Brow
 
-A generic background thread worker for shipping events via https to some API backend.
+A generic background thread worker for shipping events via https to some API backend. It'll get events to your API by the sweat of its brow.
 
 I've been wanting to build something like this for a while. This might be a terrible start. But its a start.
 
@@ -36,14 +36,13 @@ client = Brow::Client.new({
 })
 
 50.times do |n|
-  client.record({
+  client.push({
     number: n,
     now: Time.now.utc,
   })
 end
 
 # batch of 50 events sent to api url above as json
-client.flush
 ```
 
 ## Development

data/examples/basic.rb

@@ -1,14 +1,12 @@
 require_relative "../lib/brow"
 
 client = Brow::Client.new({
-  url: "https://requestbin.net/r/rna67for",
+  url: "https://requestbin.net/r/4f09194m",
 })
 
-50.times do |n|
-  client.record({
+150.times do |n|
+  client.push({
     number: n,
     now: Time.now.utc,
   })
 end
-
-client.flush

data/lib/brow/client.rb

@@ -12,6 +12,9 @@ module Brow
     # Private: Default # of items that can be in queue before we start dropping data.
     MAX_QUEUE_SIZE = 10_000
 
+    # Private: Default number of seconds to wait to shutdown worker thread.
+    SHUTDOWN_TIMEOUT = 5
+
     # Public: Create a new instance of a client.
     #
     # options - The Hash of options.
@@ -22,13 +25,17 @@ module Brow
 
       @worker_thread = nil
       @worker_mutex = Mutex.new
+      @pid = Process.pid
       @test = options[:test]
       @max_queue_size = options[:max_queue_size] || MAX_QUEUE_SIZE
       @logger = options.fetch(:logger) { Brow.logger }
       @queue = options.fetch(:queue) { Queue.new }
       @worker = options.fetch(:worker) { Worker.new(@queue, options) }
+      @shutdown_timeout = options.fetch(:shutdown_timeout) { SHUTDOWN_TIMEOUT }
 
-      at_exit { @worker_thread && @worker_thread[:should_exit] = true }
+      if options.fetch(:shutdown_automatically, true)
+        at_exit { shutdown }
+      end
     end
 
     # Public: Synchronously waits until the worker has flushed the queue.
@@ -37,22 +44,46 @@ module Brow
     # specifically exit.
     def flush
       while !@queue.empty? || @worker.requesting?
-        ensure_worker_running
+        ensure_threads_alive
         sleep(0.1)
       end
     end
 
-    # Public: Enqueues the event.
+    def shutdown
+      if @worker_thread
+        begin
+          @worker_thread.join @shutdown_timeout
+        rescue => error
+          @logger.info("[brow]") { "Error shutting down worker thread: #{error.inspect}"}
+        end
+      end
+    end
+
+    # Public: Enqueues an event to eventually be transported to backend service.
     #
     # event - The Hash of event data.
     #
     # Returns Boolean of whether the item was added to the queue.
-    def record(event)
-      raise ArgumentError, "event must be a Hash" unless event.is_a?(Hash)
+    def push(item)
+      raise ArgumentError, "item must be a Hash" unless item.is_a?(Hash)
+
+      item = Brow::Utils.symbolize_keys(item)
+      item = Brow::Utils.isoify_dates(item)
 
-      event = Brow::Utils.symbolize_keys(event)
-      event = Brow::Utils.isoify_dates(event)
-      enqueue event
+      if @test
+        test_queue << item
+        return true
+      end
+
+      ensure_threads_alive
+
+      if @queue.length < @max_queue_size
+        @queue << item
+        true
+      else
+        @logger.warn("[brow]") { "Queue is full, dropping events. The :max_queue_size configuration parameter can be increased to prevent this from happening." }
+        false
+      end
     end
 
     # Public: Returns the number of messages in the queue.
@@ -61,7 +92,7 @@ module Brow
     end
 
     # Public: For test purposes only. If test: true is passed to #initialize
-    # then all recording of events will go to test queue in memory so they can
+    # then all pushing of events will go to test queue in memory so they can
     # be verified with assertions.
     def test_queue
       unless @test
@@ -73,36 +104,34 @@ module Brow
 
     private
 
-    # Private: Enqueues the event.
-    #
-    # Returns Boolean of whether the item was added to the queue.
-    def enqueue(action)
-      if @test
-        test_queue << action
-        return true
-      end
-
-      if @queue.length < @max_queue_size
-        @queue << action
-        ensure_worker_running
+    def forked?
+      @pid != Process.pid
+    end
 
-        true
-      else
-        @logger.warn 'Queue is full, dropping events. The :max_queue_size configuration parameter can be increased to prevent this from happening.'
-        false
-      end
+    def ensure_threads_alive
+      reset if forked?
+      ensure_worker_running
     end
 
     def ensure_worker_running
-      return if worker_running?
-      @worker_mutex.synchronize do
+      # If another thread is starting worker thread, then return early so this
+      # thread can enqueue and move on with life.
+      return unless @worker_mutex.try_lock
+
+      begin
         return if worker_running?
-        @worker_thread = Thread.new do
-          @worker.run
-        end
+        @worker_thread = Thread.new { @worker.run }
+      ensure
+        @worker_mutex.unlock
       end
     end
 
+    def reset
+      @pid = Process.pid
+      @worker_mutex.unlock if @worker_mutex.locked?
+      @queue.clear
+    end
+
     def worker_running?
       @worker_thread && @worker_thread.alive?
     end

data/lib/brow/message_batch.rb

@@ -41,7 +41,7 @@ module Brow
       message_json_size = message_json.bytesize
 
       if message_too_big?(message_json_size)
-        @logger.error('a message exceeded the maximum allowed size')
+        @logger.error("[brow]") { 'a message exceeded the maximum allowed size' }
       else
         @messages << message
         @json_size += message_json_size + 1 # One byte for the comma

data/lib/brow/transport.rb

@@ -10,6 +10,8 @@ require_relative 'backoff_policy'
 module Brow
   class Transport
     RETRIES = 10
+    READ_TIMEOUT = 8
+    OPEN_TIMEOUT = 4
     HEADERS = {
       "Accept" => "application/json",
       "Content-Type" => "application/json",
@@ -18,8 +20,6 @@ module Brow
       "Client-Language-Version" => "#{RUBY_VERSION} p#{RUBY_PATCHLEVEL} (#{RUBY_RELEASE_DATE})",
       "Client-Platform" => RUBY_PLATFORM,
       "Client-Engine" => defined?(RUBY_ENGINE) ? RUBY_ENGINE : "",
-      "Client-Pid" => Process.pid.to_s,
-      "Client-Thread" => Thread.current.object_id.to_s,
       "Client-Hostname" => Socket.gethostname,
     }
 
@@ -39,32 +39,29 @@ module Brow
 
       @logger = options.fetch(:logger) { Brow.logger }
       @backoff_policy = options.fetch(:backoff_policy) {
-        Brow::BackoffPolicy.new
+        Brow::BackoffPolicy.new(options)
       }
 
       @http = Net::HTTP.new(@uri.host, @uri.port)
       @http.use_ssl = @uri.scheme == "https"
-      @http.read_timeout = options[:read_timeout] || 8
-      @http.open_timeout = options[:open_timeout] || 4
+      @http.read_timeout = options[:read_timeout] || READ_TIMEOUT
+      @http.open_timeout = options[:open_timeout] || OPEN_TIMEOUT
     end
 
     # Sends a batch of messages to the API
     #
     # @return [Response] API response
     def send_batch(batch)
-      @logger.debug("Sending request for #{batch.length} items")
+      @logger.debug("[brow]") { "Sending request for #{batch.length} items" }
 
       last_response, exception = retry_with_backoff(@retries) do
         response = send_request(batch)
-        status_code = response.code.to_i
-        should_retry = should_retry_request?(status_code, response.body)
-        @logger.debug("Response status code: #{status_code}")
-
-        [Response.new(status_code, nil), should_retry]
+        @logger.debug("[brow]") { "Response: status=#{response.code}, body=#{response.body}" }
+        [Response.new(response.code.to_i, nil), retry?(response)]
       end
 
       if exception
-        @logger.error(exception.message)
+        @logger.error("[brow]") { exception.message }
         exception.backtrace.each { |line| @logger.error(line) }
         Response.new(-1, exception.to_s)
       else
@@ -79,18 +76,19 @@ module Brow
 
     private
 
-    def should_retry_request?(status_code, body)
+    def retry?(response)
+      status_code = response.code.to_i
       if status_code >= 500
         # Server error. Retry and log.
-        @logger.info("Server error: status=#{status_code}, body=#{body}")
+        @logger.info("[brow]") { "Server error: status=#{status_code}, body=#{response.body}" }
         true
       elsif status_code == 429
-        # Rate limited
-        @logger.info "Rate limit error"
+        # Rate limited. Retry and log.
+        @logger.info("[brow]") { "Rate limit error: body=#{response.body}" }
         true
       elsif status_code >= 400
         # Client error. Do not retry, but log.
-        @logger.error("Client error: status=#{status_code}, body=#{body}")
+        @logger.error("[brow]") { "Client error: status=#{status_code}, body=#{response.body}" }
         false
       else
         false
@@ -112,13 +110,13 @@ module Brow
         result, should_retry = yield
         return [result, nil] unless should_retry
       rescue StandardError => error
-        @logger.debug "Request error: #{error}"
+        @logger.debug("[brow]") {  "Request error: #{error}" }
         should_retry = true
         caught_exception = error
       end
 
       if should_retry && (retries_remaining > 1)
-        @logger.debug("Retrying request, #{retries_remaining} retries left")
+        @logger.debug("[brow]") { "Retrying request, #{retries_remaining} retries left" }
         sleep(@backoff_policy.next_interval.to_f / 1000)
         retry_with_backoff(retries_remaining - 1, &block)
       else
@@ -126,12 +124,15 @@ module Brow
       end
     end
 
-    # Sends a request for the batch, returns [status_code, body]
     def send_request(batch)
-      payload = batch.to_json
-      @http.start unless @http.started? # Maintain a persistent connection
-      request = Net::HTTP::Post.new(@uri.path, @headers)
-      @http.request(request, payload)
+      headers = {
+        "Client-Pid" => Process.pid.to_s,
+        "Client-Thread" => Thread.current.object_id.to_s,
+      }.merge(@headers)
+
+      @http.start unless @http.started?
+      request = Net::HTTP::Post.new(@uri.path, headers)
+      @http.request(request, batch.to_json)
     end
   end
 end

data/lib/brow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Brow
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/brow/worker.rb

@@ -28,7 +28,6 @@ module Brow
       options = Brow::Utils.symbolize_keys(options)
       @on_error = options[:on_error] || DEFAULT_ON_ERROR
       @transport = options.fetch(:transport) { Transport.new(options) }
-      @logger = options.fetch(:logger) { Brow.logger }
       @batch = options.fetch(:batch) { MessageBatch.new(max_size: options[:batch_size]) }
     end
 

data/lib/brow.rb

@@ -10,13 +10,11 @@ module Brow
   def self.logger
     return @logger if @logger
 
-    base_logger = if defined?(Rails)
+    @logger = if defined?(Rails)
       Rails.logger
     else
       Logger.new(STDOUT)
     end
-
-    @logger = PrefixedLogger.new(base_logger, "[brow]")
   end
 
   # Public: Sets the logger instance to use for logging things.
@@ -26,4 +24,3 @@ module Brow
 end
 
 require_relative "brow/client"
-require_relative "brow/prefixed_logger"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: brow
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - John Nunemaker
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-20 00:00:00.000000000 Z
+date: 2021-10-26 00:00:00.000000000 Z
 dependencies: []
 description: 
 email:
@@ -30,7 +30,6 @@ files:
 - lib/brow/backoff_policy.rb
 - lib/brow/client.rb
 - lib/brow/message_batch.rb
-- lib/brow/prefixed_logger.rb
 - lib/brow/response.rb
 - lib/brow/test_queue.rb
 - lib/brow/transport.rb

data/lib/brow/prefixed_logger.rb

@@ -1,25 +0,0 @@
-module Brow
-  # Internal: Wraps an existing logger and adds a prefix to all messages.
-  class PrefixedLogger
-    def initialize(logger, prefix)
-      @logger = logger
-      @prefix = prefix
-    end
-
-    def debug(message)
-      @logger.debug("#{@prefix} #{message}")
-    end
-
-    def info(message)
-      @logger.info("#{@prefix} #{message}")
-    end
-
-    def warn(message)
-      @logger.warn("#{@prefix} #{message}")
-    end
-
-    def error(message)
-      @logger.error("#{@prefix} #{message}")
-    end
-  end
-end