brow 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94b1026b806df0a58a01aa0bc911ca756251dc5d0de510ae1c12d42665b00559
-  data.tar.gz: 55b2b5454086ac78e22681ced81bb6d500dd9c6537ebfcf67bbc9a445a2d9a79
+  metadata.gz: 6b51fe73e4a857aa33f3546cf5d7f0e434f467c19e585e00e8b7a5535765d0b4
+  data.tar.gz: b36b39d1293d6b32940b5f9306c7a679c6438cc76377ea376560223343c3e6e3
 SHA512:
-  metadata.gz: 2d23c1ff291ca611a65b1357392e2f4275502fc3467a629e85ff631f7585050095ef4cc86f0b86981fb0e7e790ce709bd2a0e2b511f0e706954cec6221ae40db
-  data.tar.gz: 884ffd8cebbcaf2681a1347a1115892676c86de12dcef693e5364531e37cef0d3700343cb8e4d2393270003c541d9efbd0e1c427fef904b66f475eb81ece2246
+  metadata.gz: 7bea156ce56ed4bc49688db5dfd9ce810fcbc6e6a18513445dc734792374b0831063e7ac831c9a1eed4f50459d95cef38181afa3071e5336e1f671a576844938
+  data.tar.gz: ad5cc322c41f678f7de7df6add329defc9c90789fbe2a71df62156d58500a3b65648dacf0bc5bdeb0d986a5d2f542cccceb6e85dd55ef9b679c23a18f81ffc90

data/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2021-11-04
+
+### Added
+
+- Allow configuring most options from ENV variables by default (fb7819b0237a81e573677f3050446a4f41e8fb47).
+- Extra early return to avoid mutex lock if thread is alive (ac7dcfe54ee83b18e0df5ab3778a077584c843bd).
+- Validation on many of the configuration options (c50b11a2917272a87937f8aa86007816a87c63a2 and 07e2581397f870249a347d4d68e4fce172d33cef).
+
+### Changed
+
+- Stop stringifying keys. Just enqueue whatever is passed and let JSON do the rest (2e63d5328e048f0fad9fc41ca0935f97fb5ada2f).
+- A bunch of test stuff to make them faster and less flaky.
+
 ## [0.3.0] - 2021-10-29
 
 https://github.com/jnunemaker/brow/pull/4

data/Gemfile

@@ -8,7 +8,7 @@ gem "minitest", "~> 5.0"
 gem "maxitest", "~> 4.1"
 gem "minitest-heat", "~> 0.0"
 gem "webmock", "~> 3.10.0"
-gem "rack", "~> 2.2.3"
+gem "climate_control", "~> 0.2.0"
 
 group(:guard) do
   gem "guard", "~> 2.18.0"

data/Guardfile

@@ -32,6 +32,7 @@ guard :minitest do
   watch(%r{^test/(.*)\/?(.*)_test\.rb$})
   watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/#{m[1]}#{m[2]}_test.rb" }
   watch(%r{^test/test_helper\.rb$})      { 'test' }
+  watch(%r{^test/support/fake_server\.rb$})      { 'test' }
 
   # with Minitest::Spec
   # watch(%r{^spec/(.*)_spec\.rb$})

data/examples/basic.rb

@@ -1,20 +1,14 @@
 require_relative "../lib/brow"
+require_relative "echo_server"
 
 client = Brow::Client.new({
-  url: "https://requestbin.net/r/2bp3p3vn",
+  url: "http://localhost:#{EchoServer.instance.port}",
   batch_size: 10,
 })
 
 5.times do |n|
-  client.push(n: n, parent: true)
+  client.push({
+    n: n,
+    now: Time.now.utc,
+  })
 end
-
-pid = fork {
-  15.times do |n|
-    client.push({
-      number: n,
-      now: Time.now.utc,
-    })
-  end
-}
-Process.waitpid pid, 0

data/examples/echo_server.rb

@@ -0,0 +1,51 @@
+# Usage: bundle exec ruby examples/echo_server.rb
+#
+# By default this starts in thread that other example scripts can use.
+#
+# By setting FOREGROUND=1, this will run in the foreground instead of
+# background thread.
+#
+# FOREGROUND=1 bundle exec ruby examples/echo_server.rb
+require "socket"
+require "thread"
+require "logger"
+require "json"
+require "rack"
+require "rack/handler/webrick"
+
+class EchoServer
+  include Singleton
+
+  attr_reader :port, :thread
+
+  def initialize
+    @logger = Logger.new(STDOUT)
+    @logger.level = Logger::INFO
+    @port = ENV.fetch("PORT", 9999)
+    @started = false
+
+    @server = WEBrick::HTTPServer.new({
+      Port: @port,
+      StartCallback: -> { @started = true },
+      Logger: WEBrick::Log.new(@logger, WEBrick::Log::INFO),
+      AccessLog: [
+        [@logger, WEBrick::AccessLog::COMMON_LOG_FORMAT],
+      ],
+    })
+
+    @server.mount '/', Rack::Handler::WEBrick, ->(env) {
+      request = Rack::Request.new(env)
+      @logger.debug JSON.parse(request.body.read).inspect
+      [200, {}, [""]]
+    }
+
+    @thread = Thread.new { @server.start }
+    Timeout.timeout(10) { :wait until @started }
+  end
+end
+
+EchoServer.instance
+
+if ENV.fetch("FOREGROUND", "0") == "1"
+  EchoServer.instance.thread.join
+end

data/examples/forked.rb

@@ -0,0 +1,20 @@
+require_relative "../lib/brow"
+require_relative "echo_server"
+
+client = Brow::Client.new({
+  url: "http://localhost:#{EchoServer.instance.port}",
+  batch_size: 10,
+})
+
+client.push({
+  now: Time.now.utc,
+  parent: true,
+})
+
+pid = fork {
+  client.push({
+    now: Time.now.utc,
+    child: true,
+  })
+}
+Process.waitpid pid, 0

data/examples/long_running.rb

@@ -0,0 +1,32 @@
+require_relative "../lib/brow"
+
+port = ENV.fetch("PORT") { 9999 }
+
+if ENV.fetch("START_SERVER", "1") == "1"
+  require_relative "echo_server"
+  port = EchoServer.instance.port
+end
+
+Brow.logger = Logger.new(STDOUT)
+Brow.logger.level = Logger::INFO
+
+client = Brow::Client.new({
+  url: "http://localhost:#{port}",
+  batch_size: 1_000,
+})
+
+running = true
+
+trap("INT") {
+  puts "Shutting down"
+  running = false
+}
+
+while running
+  rand(10_000).times { client.push("foo" => "bar") }
+
+  puts "Queue size: #{client.worker.queue.size}"
+
+  # Pretend to work
+  sleep(rand)
+end

data/lib/brow/backoff_policy.rb

@@ -32,10 +32,30 @@ module Brow
     #   :randomization_factor - The randomization factor to use to create a range
     #                           around the retry interval.
     def initialize(options = {})
-      @min_timeout_ms = options[:min_timeout_ms] || MIN_TIMEOUT_MS
-      @max_timeout_ms = options[:max_timeout_ms] || MAX_TIMEOUT_MS
-      @multiplier = options[:multiplier] || MULTIPLIER
-      @randomization_factor = options[:randomization_factor] || RANDOMIZATION_FACTOR
+      @min_timeout_ms = options.fetch(:min_timeout_ms) {
+        ENV.fetch("BROW_BACKOFF_MIN_TIMEOUT_MS", MIN_TIMEOUT_MS).to_i
+      }
+      @max_timeout_ms = options.fetch(:max_timeout_ms) {
+        ENV.fetch("BROW_BACKOFF_MAX_TIMEOUT_MS", MAX_TIMEOUT_MS).to_i
+      }
+      @multiplier = options.fetch(:multiplier) {
+        ENV.fetch("BROW_BACKOFF_MULTIPLIER", MULTIPLIER).to_f
+      }
+      @randomization_factor = options.fetch(:randomization_factor) {
+        ENV.fetch("BROW_BACKOFF_RANDOMIZATION_FACTOR", RANDOMIZATION_FACTOR).to_f
+      }
+
+      unless @min_timeout_ms >= 0
+        raise ArgumentError, ":min_timeout_ms must be >= 0 but was #{@min_timeout_ms.inspect}"
+      end
+
+      unless @max_timeout_ms >= 0
+        raise ArgumentError, ":max_timeout_ms must be >= 0 but was #{@max_timeout_ms.inspect}"
+      end
+
+      unless @min_timeout_ms <= max_timeout_ms
+        raise ArgumentError, ":min_timeout_ms (#{@min_timeout_ms.inspect}) must be <= :max_timeout_ms (#{@max_timeout_ms.inspect})"
+      end
 
       @attempts = 0
     end

data/lib/brow/message_batch.rb

@@ -25,11 +25,18 @@ module Brow
     def_delegators :@messages, :size
     def_delegators :@messages, :count
 
-    attr_reader :uuid, :json_size
+    attr_reader :uuid, :json_size, :max_size
 
     def initialize(options = {})
       clear
-      @max_size = options[:max_size] || MAX_SIZE
+      @max_size = options.fetch(:max_size) {
+        ENV.fetch("BROW_BATCH_SIZE", MAX_SIZE).to_i
+      }
+
+      unless @max_size > 0
+        raise ArgumentError, ":max_size must be > 0 but was #{@max_size.inspect}"
+      end
+
       @logger = options.fetch(:logger) { Brow.logger }
     end
 

data/lib/brow/transport.rb

@@ -3,6 +3,7 @@
 require 'net/http'
 require 'net/https'
 require 'json'
+require 'set'
 
 require_relative 'response'
 require_relative 'backoff_policy'
@@ -18,20 +19,40 @@ module Brow
     # Private: Default open timeout on requests.
     OPEN_TIMEOUT = 4
 
+    # Private: Default write timeout on requests.
+    WRITE_TIMEOUT = 4
+
+    # Private: URL schemes that this transport supports.
+    VALID_HTTP_SCHEMES = Set["http", "https"].freeze
+
     # Private
     attr_reader :url, :headers, :retries, :logger, :backoff_policy, :http
 
     def initialize(options = {})
-      @url = options[:url] || raise(ArgumentError, ":url is required to be present so we know where to send batches")
+      @url = options.fetch(:url) {
+        ENV.fetch("BROW_URL") {
+          raise ArgumentError, ":url is required to be present so we know where to send batches"
+        }
+      }
       @uri = URI.parse(@url)
 
+      unless VALID_HTTP_SCHEMES.include?(@uri.scheme)
+        raise ArgumentError, ":url was must be http(s) scheme but was #{@uri.scheme.inspect}"
+      end
+
       # Default path if people forget a slash.
       if @uri.path.nil? || @uri.path.empty?
         @uri.path = "/"
       end
 
       @headers = options[:headers] || {}
-      @retries = options[:retries] || RETRIES
+      @retries = options.fetch(:retries) {
+        ENV.fetch("BROW_RETRIES", RETRIES).to_i
+      }
+
+      unless @retries >= 0
+        raise ArgumentError, ":retries must be >= to 0 but was #{@retries.inspect}"
+      end
 
       @logger = options.fetch(:logger) { Brow.logger }
       @backoff_policy = options.fetch(:backoff_policy) {
@@ -40,8 +61,25 @@ module Brow
 
       @http = Net::HTTP.new(@uri.host, @uri.port)
       @http.use_ssl = @uri.scheme == "https"
-      @http.read_timeout = options[:read_timeout] || READ_TIMEOUT
-      @http.open_timeout = options[:open_timeout] || OPEN_TIMEOUT
+
+      read_timeout = options.fetch(:read_timeout) {
+        ENV.fetch("BROW_READ_TIMEOUT", READ_TIMEOUT).to_f
+      }
+      @http.read_timeout = read_timeout if read_timeout
+
+      open_timeout = options.fetch(:open_timeout) {
+        ENV.fetch("BROW_OPEN_TIMEOUT", OPEN_TIMEOUT).to_f
+      }
+      @http.open_timeout = open_timeout if open_timeout
+
+      if RUBY_VERSION >= '2.6.0'
+        write_timeout = options.fetch(:write_timeout) {
+          ENV.fetch("BROW_WRITE_TIMEOUT", WRITE_TIMEOUT).to_f
+        }
+        @http.write_timeout = write_timeout if write_timeout
+      else
+        Kernel.warn("Warning: option :write_timeout requires Ruby version 2.6.0 or later")
+      end
     end
 
     # Sends a batch of messages to the API
@@ -128,7 +166,7 @@ module Brow
       headers = {
         "Accept" => "application/json",
         "Content-Type" => "application/json",
-        "User-Agent" => "brow-ruby/#{Brow::VERSION}",
+        "User-Agent" => "Brow v#{Brow::VERSION}",
         "Client-Language" => "ruby",
         "Client-Language-Version" => "#{RUBY_VERSION} p#{RUBY_PATCHLEVEL} (#{RUBY_RELEASE_DATE})",
         "Client-Platform" => RUBY_PLATFORM,

data/lib/brow/utils.rb

@@ -13,13 +13,6 @@ module Brow
       end
     end
 
-    # Internal: Return a new hash with keys converted to strings
-    def stringify_keys(hash)
-      hash.each_with_object({}) do |(k, v), memo|
-        memo[k.to_s] = v
-      end
-    end
-
     # Internal: Returns a new hash with all the date values in the into
     # iso8601 strings
     def isoify_dates(hash)

data/lib/brow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Brow
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/brow/worker.rb

@@ -32,8 +32,8 @@ module Brow
     # The worker continuously takes messages off the queue and makes requests to
     # the api.
     #
-    # queue   - Queue synchronized between client and worker
     # options - The Hash of worker options.
+    #   :queue - Queue synchronized between client and worker
     #   :on_error - Proc of what to do on an error.
     #   :batch_size - Fixnum of how many items to send in a batch.
     #   :transport - The Transport object to deliver batches.
@@ -42,6 +42,8 @@ module Brow
     #            via Transport.
     #   :shutdown_timeout - The number of seconds to wait for the worker thread
     #                       to join when shutting down.
+    #   :start_automatically - Should the client start the worker thread
+    #                          automatically and keep it running.
     #   :shutdown_automatically - Should the client shutdown automatically or
     #                             manually. If true, shutdown is automatic. If
     #                             false, you'll need to handle this on your own.
@@ -52,11 +54,32 @@ module Brow
       @mutex = Mutex.new
       options = Brow::Utils.symbolize_keys(options)
       @on_error = options[:on_error] || DEFAULT_ON_ERROR
-      @batch_size = options[:batch_size]
-      @max_queue_size = options.fetch(:max_queue_size) { MAX_QUEUE_SIZE }
       @logger = options.fetch(:logger) { Brow.logger }
       @transport = options.fetch(:transport) { Transport.new(options) }
-      @shutdown_timeout = options.fetch(:shutdown_timeout) { SHUTDOWN_TIMEOUT }
+
+      @batch_size = options.fetch(:batch_size) {
+        ENV.fetch("BROW_BATCH_SIZE", MessageBatch::MAX_SIZE).to_i
+      }
+      @max_queue_size = options.fetch(:max_queue_size) {
+        ENV.fetch("BROW_MAX_QUEUE_SIZE", MAX_QUEUE_SIZE).to_i
+      }
+      @shutdown_timeout = options.fetch(:shutdown_timeout) {
+        ENV.fetch("BROW_SHUTDOWN_TIMEOUT", SHUTDOWN_TIMEOUT).to_f
+      }
+
+      if @batch_size <= 0
+        raise ArgumentError, ":batch_size must be greater than 0"
+      end
+
+      if @max_queue_size <= 0
+        raise ArgumentError, ":max_queue_size must be greater than 0"
+      end
+
+      if @shutdown_timeout <= 0
+        raise ArgumentError, ":shutdown_timeout must be greater than 0"
+      end
+
+      @start_automatically = options.fetch(:start_automatically, true)
 
       if options.fetch(:shutdown_automatically, true)
         at_exit { stop }
@@ -65,9 +88,9 @@ module Brow
 
     def push(data)
       raise ArgumentError, "data must be a Hash" unless data.is_a?(Hash)
-      start
+      start if @start_automatically
 
-      data = Utils.isoify_dates(Utils.stringify_keys(data))
+      data = Utils.isoify_dates(data)
 
       if queue.length < max_queue_size
         queue << data
@@ -108,6 +131,7 @@ module Brow
 
         case message
         when SHUTDOWN
+          logger.info("[brow]") { "Worker shutting down" }
           send_batch(batch) unless batch.empty?
           break
         else
@@ -131,12 +155,15 @@ module Brow
     end
 
     def ensure_worker_running
+      # Return early if thread is alive and avoid the mutex lock and unlock.
+      return if thread_alive?
+
       # If another thread is starting worker thread, then return early so this
       # thread can enqueue and move on with life.
       return unless mutex.try_lock
 
       begin
-        return if @thread && @thread.alive?
+        return if thread_alive?
         @thread = Thread.new { run }
         logger.debug("[brow]") { "Worker thread [#{@thread.object_id}] started" }
       ensure
@@ -144,6 +171,10 @@ module Brow
       end
     end
 
+    def thread_alive?
+      @thread && @thread.alive?
+    end
+
     def reset
       @pid = Process.pid
       mutex.unlock if mutex.locked?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: brow
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - John Nunemaker
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-29 00:00:00.000000000 Z
+date: 2021-11-04 00:00:00.000000000 Z
 dependencies: []
 description: 
 email:
@@ -26,6 +26,9 @@ files:
 - bin/console
 - bin/setup
 - examples/basic.rb
+- examples/echo_server.rb
+- examples/forked.rb
+- examples/long_running.rb
 - lib/brow.rb
 - lib/brow/backoff_policy.rb
 - lib/brow/client.rb