rack-influxdb 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 692f56fbc15ce9112783e0a51f3a323e32bcb1063db207d38684d66ef11f0c45
-  data.tar.gz: b6d827637d9b7162a69b04682846ffbb372bce98af6b2fa547d6d2a0dd967864
+  metadata.gz: 56285fb58ca4b8a2a89e99d44d56226ef01fe04617c2211eb23036f5f913dd9c
+  data.tar.gz: 61c49b66c095a40936e1014ebb828b6aafe031d60b35a452bbb90ebe661bb2a3
 SHA512:
-  metadata.gz: 12047f80e2e445e38b03304590ab8b77d197d6ab6061f3e68bef009dca49e40aad006cf2eac593ec33012f9af2dd3054fda4c9ab84eedc1fc9cb4ef98b33d02b
-  data.tar.gz: 31cdc1e24c9e4931e545d93139a7fa4709fbc03597f3a6891f60bca98cbd695b2422fab94779af62f4f5716bf636c26891a734169fd31af69654cc3fd6b23145
+  metadata.gz: 0c477dc8d0368309191d354c5efcdc2d9892b5a1959eba15d04556d2a1bf2c204e2a79380928bcbd523cb1f1cb82b74aac13bc89e586462fb1ec93f790e94a25
+  data.tar.gz: c773adb6e3477eb96477f5e88afc579336b50657a12dcf066a410baab9273cb582bdab9490f283848fe58a08ced59b93ea279587340c01269a749dc615f30bea

data/README.md

@@ -1,6 +1,12 @@
 # rack-influxdb
+![Github Chimpy](https://github.com/user-attachments/assets/4f7c612e-2de7-45b4-bee7-dd54ab3e1d51)
 
 [![Continuous Integration](https://github.com/heychimpy/rack-influxdb/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/heychimpy/rack-influxdb/actions/workflows/test.yml)
+[![codecov](https://codecov.io/gh/heychimpy/rack-influxdb/graph/badge.svg?token=L9NVAKCQWU)](https://codecov.io/gh/heychimpy/rack-influxdb)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop/rubocop)
+![Reek Badge](https://img.shields.io/badge/code%20quality-reek-brightgreen?style=flat-square)
+![Brakeman Badge](https://img.shields.io/badge/security-brakeman-brightgreen?style=flat-square)
 
 This gem provides a Rack middleware for [InfluxDB](https://github.com/influxdata/influxdb).
 

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 require 'bundler/setup'
 require 'bundler/gem_tasks'

data/lib/rack/influxdb/configuration.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 module Rack
   class InfluxDB
+    # The main configuration for the gem.
     class Configuration
       attr_accessor :name, :tags, :url, :token, :options, :write_options,
                     :fields, :handle_error
@@ -15,17 +18,24 @@ module Rack
         @tags = {}
         @url = ''
         @token = ''
-        @fields = lambda { |env, response| return {} }
-        @handle_error = lambda { |err| return }
-        @options = {
+        @fields = ->(_env, _response) { {} }
+        @handle_error = ->(_err) { nil }
+        @options = default_options
+        @write_options = default_write_options
+      end
+
+      def default_options
+        {
           org: '',
           bucket: '',
           precision: InfluxDB2::WritePrecision::MILLISECOND
         }
+      end
 
-        @write_options = InfluxDB2::WriteOptions.new(
+      def default_write_options
+        InfluxDB2::WriteOptions.new(
           write_type: InfluxDB2::WriteType::BATCHING,
-          batch_size: 100,
+          batch_size: 100
         )
       end
     end

data/lib/rack/influxdb/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Rack
   class InfluxDB
-    VERSION = '0.1.0'
+    VERSION = '0.2.0'
   end
 end

data/lib/rack/influxdb.rb

@@ -1,11 +1,18 @@
+# frozen_string_literal: true
+
 require 'influxdb-client'
 require 'rack/influxdb/configuration'
 
 module Rack
+  # The InfluxDB rack middleware class.
   class InfluxDB
-    class << self
-      attr_reader :configuration
+    # Caps how many computed points can be buffered while waiting to be
+    # written. If InfluxDB becomes slow or unreachable, the queue fills up
+    # and further points are dropped (see #enqueue) instead of growing
+    # without bound or blocking requests.
+    DEFAULT_QUEUE_SIZE = 1_000
 
+    class << self
       def configuration
         @configuration ||= Rack::InfluxDB::Configuration.new
       end
@@ -17,12 +24,28 @@ module Rack
 
     def initialize(app)
       @app = app
+      @queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
+      @worker_mutex = Mutex.new
+      @worker = nil
+
+      # Flush whatever is left in the queue when the process exits (e.g. on
+      # a graceful Puma shutdown), so the last few requests before shutdown
+      # aren't silently lost. This never blocks a request, only exit.
+      at_exit { shutdown }
     end
 
     def call(env)
       @app.call(env).tap { |response| write_request(env, response) }
     end
 
+    # Stops accepting new points and waits (briefly) for the worker to
+    # drain whatever is already queued. Safe to call multiple times and
+    # safe to call even if the worker was never started.
+    def shutdown
+      @queue.close unless @queue.closed?
+      @worker&.join(5)
+    end
+
     private
 
     def write_request(env, response)
@@ -31,31 +54,89 @@ module Rack
       # environment.
       return if config.token.to_s.empty?
 
-      # Run the write logic inside a thread so we don't slow down
-      # main request.
-      Thread.new do
-        begin
-          InfluxDB2::Client.use(config.url, config.token, **config.options) do |c|
-            write_api = c.create_write_api(write_options: config.write_options)
-            write_api.write(data: point(env, response))
-          end
-        rescue => e
-          config.handle_error.call(e)
-        end
-      end
-    rescue => e
+      # Compute the point on the request thread so the queue - and the
+      # worker that reads from it - never has to hold a reference to the
+      # Rack env or response. Only the resulting Hash is handed off.
+      enqueue(point(env, response))
+    rescue StandardError => e
       # Let the app decide what needs to be done when an error occurs.
       config.handle_error.call(e)
     end
 
+    # Rather than spawning a `Thread.new` (and a new client/WriteApi) for
+    # every single request, requests push their computed point onto a
+    # queue that a single long-lived worker thread drains. This avoids the
+    # per-request cost of thread creation/teardown and the memory growth
+    # that comes from many short-lived threads each closing over a Rack
+    # env and response.
+    def enqueue(data)
+      start_worker
+      # Non-blocking push: if the queue is full (InfluxDB is slow/down) we
+      # drop the point instead of blocking the request or growing memory
+      # without bound.
+      @queue.push(data, true)
+    rescue ClosedQueueError, ThreadError
+      nil
+    end
+
+    # Lazily starts exactly one worker thread for this middleware instance.
+    #
+    # It's started lazily, on first request, rather than eagerly in
+    # `initialize`, to stay correct under Puma's `preload_app!`: the app
+    # (and this middleware) is built once in the master process, which
+    # then fork()s a child process per cluster worker. OS threads do not
+    # survive fork - only the forking thread continues to exist in the
+    # child - so a thread started eagerly in `initialize` would already be
+    # dead in every forked worker. Starting on first request instead means
+    # each process (master or forked worker) creates its own worker thread
+    # the first time it actually handles a request, so every worker
+    # process naturally ends up owning exactly one live worker thread and
+    # one queue.
+    def start_worker
+      return if @worker&.alive?
+
+      @worker_mutex.synchronize do
+        return if @worker&.alive?
+
+        @worker = Thread.new { process_queue }
+      end
+    end
+
+    # Runs on the single worker thread only. `client` and `write_api` are
+    # therefore each created exactly once, lazily, with no need for extra
+    # synchronization: nothing else ever touches them.
+    def process_queue
+      loop do
+        data = @queue.pop
+        # `pop` only returns nil once the queue has been closed (see
+        # #shutdown) and fully drained, which is our signal to stop.
+        break if data.nil?
+
+        write_api.write(data: data)
+      rescue StandardError => e
+        config.handle_error.call(e)
+      end
+    end
+
     def point(env, response)
       {
         name: config.name,
         tags: config.tags,
+        time: (Time.now.to_r * 1_000_000_000).to_i,
         fields: config.fields.call(env, response)
       }
     end
 
+    def write_api
+      @write_api ||= client.create_write_api(write_options: config.write_options)
+    end
+
+    def client
+      @client ||= InfluxDB2::Client.new(
+        config.url, config.token, **config.options
+      )
+    end
+
     def config
       Rack::InfluxDB.configuration
     end

data/spec/rack_influxdb_spec.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require_relative './spec_helper'
+require 'weakref'
 
 RSpec.describe Rack::InfluxDB do
   describe '.configuration' do
@@ -32,10 +35,6 @@ RSpec.describe Rack::InfluxDB do
   end
 
   describe '#call' do
-    before do
-      allow(Thread).to receive(:new).and_yield
-    end
-
     it 'returns the initial status code' do
       get '/'
 
@@ -49,41 +48,121 @@ RSpec.describe Rack::InfluxDB do
     end
 
     context 'when no config token is given' do
-      it 'does not call InfluxDB2::Client.use' do
-        expect(InfluxDB2::Client).not_to receive(:use)
+      it 'does not call InfluxDB2::Client.new' do
+        expect(InfluxDB2::Client).not_to receive(:new)
 
         get '/'
       end
     end
 
     context 'when config token is given' do
+      let(:conf) { described_class.configuration }
+      let(:influx_client) { double('InfluxDB2::Client') }
+      let(:write_api) { double('InfluxDB2::WriteApi') }
+      let(:write_calls) { [] }
+      let(:write_mutex) { Mutex.new }
+
       before do
-        allow(InfluxDB2::Client).to receive(:use)
+        allow(InfluxDB2::Client).to receive(:new).and_return(influx_client)
+        allow(influx_client).to receive(:create_write_api).and_return(write_api)
+        allow(write_api).to receive(:write) do |data:|
+          write_mutex.synchronize { write_calls << data }
+        end
 
         described_class.configure do |config|
-          config.token = "token"
-          config.url = "example.com"
+          config.token = 'token'
+          config.url = 'example.com'
         end
       end
 
-      let(:conf) { described_class.configuration }
-
-      it 'calls InfluxDB2::Client.use with right params' do
+      it 'calls InfluxDB2::Client.new with right params' do
         get '/'
 
+        wait_for { write_calls.any? }
+
         expect(InfluxDB2::Client)
-          .to have_received(:use)
+          .to have_received(:new)
           .with(conf.url, conf.token, conf.options)
       end
+
+      it 'writes a data point to InfluxDB' do
+        get '/'
+
+        wait_for { write_calls.any? }
+
+        expect(write_calls).to include(hash_including(name: conf.name))
+      end
+
+      it 'reuses the same write API instead of creating a new one per request' do
+        middleware = described_class.new(->(_env) { [200, {}, ['Hello World']] })
+
+        2.times { middleware.call(Rack::MockRequest.env_for('/')) }
+
+        wait_for { write_calls.size >= 2 }
+
+        expect(influx_client).to have_received(:create_write_api).once
+      end
+
+      it 'returns the response without waiting for the write to InfluxDB' do
+        allow(write_api).to receive(:write) do |data:|
+          sleep 0.3
+          write_mutex.synchronize { write_calls << data }
+        end
+
+        middleware = described_class.new(->(_env) { [200, {}, ['Hello World']] })
+
+        started_at = Time.now
+        middleware.call(Rack::MockRequest.env_for('/'))
+        elapsed = Time.now - started_at
+
+        expect(elapsed).to be < 0.1
+      end
+
+      it 'creates exactly one worker thread no matter how many requests come in' do
+        thread_count = 0
+        thread_count_mutex = Mutex.new
+        allow(Thread).to receive(:new).and_wrap_original do |original, *args, &block|
+          thread_count_mutex.synchronize { thread_count += 1 }
+          original.call(*args, &block)
+        end
+
+        middleware = described_class.new(->(_env) { [200, {}, ['Hello World']] })
+        5.times { middleware.call(Rack::MockRequest.env_for('/')) }
+
+        wait_for { write_calls.size >= 5 }
+
+        expect(thread_count).to eq(1)
+      end
+
+      it 'does not retain the Rack env after the worker has processed it' do
+        middleware = described_class.new(->(_env) { [200, {}, ['Hello World']] })
+
+        # Building the env and the WeakRef inside a lambda, rather than in
+        # local variables of the example itself, lets `env` fall out of
+        # scope (and become eligible for GC) as soon as the lambda returns.
+        weak_env = lambda do
+          env = Rack::MockRequest.env_for('/')
+          ref = WeakRef.new(env)
+          middleware.call(env)
+          ref
+        end.call
+
+        wait_for { write_calls.any? }
+        GC.start
+
+        # `weakref_alive?` is falsy (nil or false, depending on Ruby version)
+        # once the referenced object has been collected.
+        expect(weak_env.weakref_alive?).to be_falsey
+      end
     end
 
     context 'when an error occurs' do
       before do
-        allow(InfluxDB2::Client).to receive(:use).and_raise('Could not write')
+        allow(InfluxDB2::Client).to receive(:new).and_raise('Could not write')
 
         described_class.configure do |config|
-          config.token = "token"
-          config.url = "example.com"
+          config.token = 'token'
+          config.url = 'example.com'
         end
       end
 
@@ -98,14 +177,21 @@ RSpec.describe Rack::InfluxDB do
       end
 
       context 'when custom error handling is applied' do
-        before do
+        it 'passes the raised error to the custom error handler' do
+          handled_errors = []
+          handled_mutex = Mutex.new
+
           described_class.configure do |config|
-            config.handle_error = ->(e) { raise e }
+            config.handle_error = lambda do |e|
+              handled_mutex.synchronize { handled_errors << e }
+            end
           end
-        end
 
-        it 'it gets handled (re-raised)' do
-          expect { get '/' }.to raise_error('Could not write')
+          get '/'
+
+          wait_for { handled_errors.any? }
+
+          expect(handled_errors.first.message).to eq('Could not write')
         end
       end
     end

data/spec/spec_helper.rb

@@ -1,3 +1,11 @@
+# frozen_string_literal: true
+
+require 'simplecov'
+require 'simplecov-cobertura'
+
+SimpleCov.formatter = SimpleCov::Formatter::CoberturaFormatter
+SimpleCov.start
+
 require 'bundler/setup'
 
 require 'rack'
@@ -10,10 +18,23 @@ def app
     # Include the InfluxDB middleware.
     use Rack::InfluxDB
 
-    run lambda { |_env| [200, {}, ['Hello World']] }
+    run ->(_env) { [200, {}, ['Hello World']] }
   end.to_app
 end
 
+# Polls `block` until it returns truthy, or raises after `timeout` seconds.
+# Used to synchronize with work happening on the middleware's background
+# worker thread without coupling tests to sleep durations.
+def wait_for(timeout: 2)
+  deadline = Time.now + timeout
+
+  until yield
+    raise "wait_for timed out after #{timeout}s" if Time.now > deadline
+
+    sleep 0.01
+  end
+end
+
 RSpec.configure do |config|
   config.include Rack::Test::Methods
 

metadata

@@ -1,35 +1,42 @@
 --- !ruby/object:Gem::Specification
 name: rack-influxdb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Henning Vogt
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-14 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: base64
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-    - - "<"
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: influxdb-client
   requirement: !ruby/object:Gem::Requirement
@@ -44,6 +51,40 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -124,7 +165,6 @@ homepage: https://github.com/heychimpy/rack-influxdb
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -132,15 +172,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
-signing_key: 
+rubygems_version: 4.0.11
 specification_version: 4
 summary: Log HTTP requests to InfluxDB
 test_files: