rails_api_logger 0.11.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c60034742e7231e4a31d581dc71a67eb3826aa4702e0ff284c62ca74d477720
-  data.tar.gz: fb275befe7ec034fbe2cb00c55916c1836887d1a39b7195433fa7587e2692be9
+  metadata.gz: 94143c1b37ced487b63c0f994a1ce48077f2e2dbd65b525430f7015670b6faf0
+  data.tar.gz: d387bed67f8534081603c3ac48d26afcf2ae2f19be190088146389e97958cefd
 SHA512:
-  metadata.gz: 89ae643a486c5dda07badee17bfd01a226dda46ffdcc8d68638ea4ba36f3403439adb11cc1ee8b2cd5f91e59595ee34ebf199607d3d2a5a8569bafc39e3610f2
-  data.tar.gz: da1059a513f47d89fee5cc48ed1431c8b9f0c685fc55e09931378b0b4a880440ed936bebc6133399da1cfa652f98dda6acb1f37a53f644baa13cf2e9581fdf21
+  metadata.gz: af01668c07dd98ff8a959a858241a3a910c0fbec4a181795b231cd00bf99f791629060f81e528e79ce664b81780c33b64f7f5216a60c2c47d3963a9e5b9c3453
+  data.tar.gz: 99a2c9c802084adca6687b91fb76c21c0708883443913619528e132630723c6db8353f45b5cd46f99a490b57ad0cca5a6db53acb1c722ea86a8c24301b3c29a8

data/.ruby-version

@@ -1 +1 @@
-3.2.9
+3.2.11

data/.semaphore/semaphore.yml

@@ -20,6 +20,7 @@ blocks:
           - checkout --use-cache
           - cache restore
           - bundle config set path 'vendor/bundle'
+          - bundle config set clean 'true'
           - bundle install -j 4
 
           - cache store
@@ -58,7 +59,7 @@ blocks:
             - sem-service start postgres 14
             - bin/rails db:create db:migrate
             - bundle exec rspec
-        - name: tests postgres same db and separate target | rails 8.0
+        - name: tests postgres same db | rails 8.0
           env_vars:
             - name: SAME_DB
               value: "true"
@@ -70,7 +71,7 @@ blocks:
             - sem-service start postgres 14
             - bin/rails db:create db:migrate
             - bundle exec rspec
-        - name: tests postgres separate db and separate target | rails 6.1
+        - name: tests postgres separate db and separate target | rails 8.1
           env_vars:
             - name: SAME_DB
               value: "false"
@@ -81,57 +82,49 @@ blocks:
             - name: RAILS_ENV
               value: test
             - name: BUNDLE_GEMFILE
-              value: /home/semaphore/rails_api_logger/gemfiles/rails_6.1.gemfile
+              value: /home/semaphore/rails_api_logger/gemfiles/rails_8.1.gemfile
           commands:
             - sem-service start postgres 14
             - bin/rails db:create db:migrate
             - bundle exec rspec
-        - name: tests postgres separate db and separate target | rails 7.0
+        - name: tests mysql separate db and separate target | rails 8.0
           env_vars:
             - name: SAME_DB
               value: "false"
             - name: SAME_TARGET
               value: "false"
             - name: TARGET_DB
-              value: postgres
+              value: mysql
             - name: RAILS_ENV
               value: test
-            - name: BUNDLE_GEMFILE
-              value: /home/semaphore/rails_api_logger/gemfiles/rails_7.0.gemfile
           commands:
-            - sem-service start postgres 14
+            - sem-service start mysql 8.0
             - bin/rails db:create db:migrate
             - bundle exec rspec
-        - name: tests postgres separate db and separate target | rails 7.1
+        - name: tests mysql separate db and same target | rails 8.0
           env_vars:
             - name: SAME_DB
               value: "false"
             - name: SAME_TARGET
-              value: "false"
+              value: "true"
             - name: TARGET_DB
-              value: postgres
+              value: mysql
             - name: RAILS_ENV
               value: test
-            - name: BUNDLE_GEMFILE
-              value: /home/semaphore/rails_api_logger/gemfiles/rails_7.1.gemfile
           commands:
-            - sem-service start postgres 14
+            - sem-service start mysql 8.0
             - bin/rails db:create db:migrate
             - bundle exec rspec
-        - name: tests postgres separate db and separate target | rails 7.2
+        - name: tests mysql same db | rails 8.0
           env_vars:
             - name: SAME_DB
-              value: "false"
-            - name: SAME_TARGET
-              value: "false"
+              value: "true"
             - name: TARGET_DB
-              value: postgres
+              value: mysql
             - name: RAILS_ENV
               value: test
-            - name: BUNDLE_GEMFILE
-              value: /home/semaphore/rails_api_logger/gemfiles/rails_7.2.gemfile
           commands:
-            - sem-service start postgres 14
+            - sem-service start mysql 8.0
             - bin/rails db:create db:migrate
             - bundle exec rspec
         - name: tests sqlite separate db and separate target | rails 8.0

data/Appraisals

@@ -1,40 +1,15 @@
-appraise "rails-6.1" do
-  gem "concurrent-ruby", "1.3.4"
-  gem "rails", "~> 6.1.0"
-  gem "rspec-rails", "~> 6.1.0"
-  gem "sqlite3", "~> 1.7.3"
-  gem "ammeter", "~> 1.1"
-  gem "webmock", "~> 3.25.0"
-end
-
-appraise "rails-7.0" do
-  gem "concurrent-ruby", "1.3.4"
-  gem "rails", "~> 7.0.0"
-  gem "rspec-rails", "~> 7.1.0"
-  gem "sqlite3", "~> 1.7.3"
-  gem "ammeter", "~> 1.1"
-  gem "webmock", "~> 3.25.0"
-end
-
-appraise "rails-7.1" do
-  gem "rails", "~> 7.1.0"
-  gem "rspec-rails", "~> 7.1.0"
-  gem "sqlite3", "~> 1.7.3"
-  gem "ammeter", "~> 1.1"
-  gem "webmock", "~> 3.25.0"
-end
-
-appraise "rails-7.2" do
-  gem "rails", "~> 7.2.0"
+appraise "rails-8.0" do
+  gem "rails", "~> 8.0.0"
   gem "rspec-rails", "~> 7.1.0"
-  gem "sqlite3", "~> 1.7.3"
+  gem "sqlite3", "~> 2.1.0"
   gem "ammeter", "~> 1.1"
+  gem "puma"
   gem "webmock", "~> 3.25.0"
 end
 
-appraise "rails-8.0" do
-  gem "rails", "~> 8.0.0"
-  gem "rspec-rails", "~> 7.1.0"
+appraise "rails-8.1" do
+  gem "rails", "~> 8.1.0"
+  gem "rspec-rails", "~> 8.0"
   gem "sqlite3", "~> 2.1.0"
   gem "ammeter", "~> 1.1"
   gem "puma"

data/CHANGELOG.md

@@ -1,10 +1,46 @@
+# 1.0.0
+
+* **Breaking:** Drop support for Rails < 8.0 and Ruby < 3.2.
+* Logging never breaks the host app anymore. When the log can't be written (the logging database is down, a
+  validation fails, a column is too small...), the error is reported to `Rails.error` with source
+  `rails_api_logger` and the request goes on without a log. Before, the app answered 500, an outbound request was
+  not sent, or a successful outbound request raised. The errors of your own requests are still raised as before.
+* Fix a crash (500 in the host app) when a request body is not valid UTF-8, for example a file upload
+  ([#6](https://github.com/renuo/rails_api_logger/issues/6)). Request and response bodies that are not valid UTF-8
+  are now stored as `"[Binary: <size> bytes]"`. Before, such a response body was stored as `nil`.
+* The install migration creates `started_at` and `ended_at` as `datetime` instead of `timestamp`. On MySQL 8,
+  `timestamp` columns have no fractional seconds, so the duration of a request was rounded to whole seconds.
+  Existing MySQL apps can migrate their columns with:
+
+```ruby
+%i[inbound_request_logs outbound_request_logs].each do |table|
+  change_column table, :started_at, :datetime, precision: 6
+  change_column table, :ended_at, :datetime, precision: 6
+end
+```
+
+# 0.12.0
+
+* Add `client_reference` column to both `inbound_request_logs` and `outbound_request_logs`. This is a free-form
+  indexed string you can set on a log entry to correlate it with any external identifier (similar in spirit to
+  `loggable`, but without a polymorphic association).
+    * For outbound requests, pass it to the logger: `RailsApiLogger::Logger.new(client_reference: "abc-123")`.
+    * For inbound requests, call `attach_inbound_request_client_reference("abc-123")` from your controller.
+
+To upgrade an existing app, run:
+
+```
+bin/rails g rails_api_logger:upgrade_client_reference
+bin/rails db:migrate
+```
+
 # 0.11.1
 
 * Remove deprecated "require_dependency"
 
 # 0.11.0
 
-* Fix stack trace to deep in webrick
+* Fix stack trace too deep in webrick
 * Fixes for Rails < 7.1
 * Support custom primary key types in migration template
 

data/README.md

@@ -23,7 +23,7 @@ This gem creates two database tables to log the following information:
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Requires Rails 8.0 or newer. Add this line to your application's Gemfile:
 
 ```ruby
 gem 'rails_api_logger'
@@ -105,6 +105,11 @@ end
 
 This will guarantee that the log is always persisted, even in case of errors.
 
+### Errors while logging
+
+Logging never breaks your app. If a log can't be written, for example because the logging database is down,
+the error is reported to `Rails.error` with source `rails_api_logger` and the request goes on without a log.
+
 ### Database Transactions Caveats
 
 If you log your outbound requests inside of parent app transactions, your logs will not be persisted if
@@ -146,6 +151,9 @@ config.middleware.insert_before Rails::Rack::Logger, RailsApiLogger::Middleware,
                                 skip_response_body_regexp: /api\/letters/
 ```
 
+Bodies that are not valid UTF-8, such as file uploads, images or PDFs, are not stored.
+The log keeps only their size, for example `"[Binary: 2048 bytes]"`.
+
 In the implementation of your API, you can call any time `attach_inbound_request_loggable(model)`
 to attach an already persisted model to the log record.
 
@@ -174,6 +182,29 @@ to be able to access the inbound logs attached to the model.
 
 You also have `has_many_outbound_request_logs` and `has_many_request_logs` that includes both.
 
+### Client reference
+
+Both `inbound_request_logs` and `outbound_request_logs` have a `client_reference` column: a free-form indexed
+string you can set on a log entry to correlate it with any external identifier (e.g. a request id coming from an
+upstream system, an idempotency key, a tenant slug). It is independent from the `loggable` polymorphic association.
+
+For inbound requests, attach it from your controller:
+
+```ruby
+def create
+  attach_inbound_request_client_reference(request.headers["X-Request-Id"])
+  # ...
+end
+```
+
+For outbound requests, pass it when instantiating the logger:
+
+```ruby
+RailsApiLogger::Logger.new(client_reference: "ref-abc").call(uri, request) do
+  http.request(request)
+end
+```
+
 ## RailsAdmin integration
 
 We provide here some code samples to integrate the models in [RailsAdmin](https://github.com/sferik/rails_admin).
@@ -227,13 +258,13 @@ end
 
 ## Development
 
-We use Appraisals to un on different Rails versions. This is a run example:
+We use Appraisals to run on different Rails versions. This is a run example:
 
 ```bash
 export SAME_TARGET=false 
 export SAME_DB=false 
 export TARGET_DB=postgres 
-export BUNDLE_GEMFILE=/Users/alessandrorodi/RenuoWorkspace/rails_api_logger/gemfiles/rails_6.1.gemfile 
+export BUNDLE_GEMFILE=/Users/alessandrorodi/RenuoWorkspace/rails_api_logger/gemfiles/rails_8.0.gemfile 
 bundle exec rails db:create db:migrate
 bundle exec rspec
 ```

data/app/controllers/inbound_requests_logger.rb

@@ -5,6 +5,12 @@ module InboundRequestsLogger
 
   def attach_inbound_request_loggable(loggable)
     return unless request.env["INBOUND_REQUEST_LOG"].present?
-    request.env["INBOUND_REQUEST_LOG"].update(loggable: loggable) if loggable&.persisted?
+    return unless loggable&.persisted?
+    RailsApiLogger.safely { request.env["INBOUND_REQUEST_LOG"].update(loggable: loggable) }
+  end
+
+  def attach_inbound_request_client_reference(client_reference)
+    return unless request.env["INBOUND_REQUEST_LOG"].present?
+    RailsApiLogger.safely { request.env["INBOUND_REQUEST_LOG"].update(client_reference: client_reference) }
   end
 end

data/app/middlewares/rails_api_logger/middleware.rb

@@ -19,22 +19,26 @@ module RailsApiLogger
       request = ActionDispatch::Request.new(env)
       logging = log?(env, request)
       if logging
-        env["INBOUND_REQUEST_LOG"] = InboundRequestLog.from_request(request, skip_request_body: skip_request_body?(env))
+        env["INBOUND_REQUEST_LOG"] = RailsApiLogger.safely do
+          InboundRequestLog.from_request(request, skip_request_body: skip_request_body?(env))
+        end
         request.body.rewind if request.body.respond_to?(:rewind)
       end
       status, headers, body = @app.call(env)
-      if logging
-        updates = {response_code: status, ended_at: Time.current}
-        updates[:response_body] = if skip_response_body?(env)
-          "[Skipped]"
-        else
-          parsed_body(body)
-        end
-        # this usually works. let's be optimistic.
-        begin
-          env["INBOUND_REQUEST_LOG"].update_columns(updates)
-        rescue JSON::GeneratorError => _e # this can be raised by activerecord if the string is not UTF-8.
-          env["INBOUND_REQUEST_LOG"].update_columns(updates.except(:response_body))
+      if logging && env["INBOUND_REQUEST_LOG"]
+        RailsApiLogger.safely do
+          updates = {response_code: status, ended_at: Time.current}
+          updates[:response_body] = if skip_response_body?(env)
+            "[Skipped]"
+          else
+            parsed_body(body)
+          end
+          # this usually works. let's be optimistic.
+          begin
+            env["INBOUND_REQUEST_LOG"].update_columns(updates)
+          rescue JSON::GeneratorError => _e # this can be raised by activerecord if the string is not UTF-8.
+            env["INBOUND_REQUEST_LOG"].update_columns(updates.except(:response_body))
+          end
         end
       end
       [status, headers, body]
@@ -60,7 +64,6 @@ module RailsApiLogger
     end
 
     def parsed_body(body)
-      return unless body.present?
       parsable_body = if body.respond_to?(:to_ary)
         body.to_ary[0]
       elsif body.respond_to?(:body)
@@ -69,9 +72,7 @@ module RailsApiLogger
         body
       end
 
-      JSON.parse(parsable_body)
-    rescue JSON::ParserError, ArgumentError
-      parsable_body
+      RequestLog.parse_body(parsable_body).presence
     end
 
     def request_with_state_change?(request)

data/app/models/rails_api_logger/logger.rb

@@ -1,22 +1,30 @@
 module RailsApiLogger
   class Logger
-    def initialize(loggable = nil, skip_request_body: false, skip_response_body: false)
+    def initialize(loggable = nil, client_reference: nil, skip_request_body: false, skip_response_body: false)
       @loggable = loggable
+      @client_reference = client_reference
       @skip_request_body = skip_request_body
       @skip_response_body = skip_response_body
     end
 
     def call(url, request)
-      log = OutboundRequestLog.from_request(request, loggable: @loggable, skip_request_body: @skip_request_body)
+      log = RailsApiLogger.safely do
+        OutboundRequestLog.from_request(request, loggable: @loggable, client_reference: @client_reference,
+          skip_request_body: @skip_request_body)
+      end
       yield.tap do |response|
-        log.from_response(response, skip_response_body: @skip_response_body)
+        RailsApiLogger.safely { log&.from_response(response, skip_response_body: @skip_response_body) }
       end
     rescue => e
       log.response_body = {error: e.message} if log
       raise
     ensure
-      log.ended_at = Time.current
-      log.save!
+      if log
+        RailsApiLogger.safely do
+          log.ended_at = Time.current
+          log.save!
+        end
+      end
     end
   end
 end

data/app/models/rails_api_logger/request_log.rb

@@ -4,13 +4,8 @@ module RailsApiLogger
 
     connects_to(**RailsApiLogger.connects_to) if RailsApiLogger.connects_to
 
-    if Gem::Version.new(Rails.version) >= Gem::Version.new("7.1")
-      serialize :request_body, coder: JSON
-      serialize :response_body, coder: JSON
-    else
-      serialize :request_body, JSON
-      serialize :response_body, JSON
-    end
+    serialize :request_body, coder: JSON
+    serialize :response_body, coder: JSON
 
     belongs_to :loggable, optional: true, polymorphic: true
 
@@ -19,19 +14,29 @@ module RailsApiLogger
     validates :method, presence: true
     validates :path, presence: true
 
-    def self.from_request(request, loggable: nil, skip_request_body: false)
+    def self.from_request(request, loggable: nil, client_reference: nil, skip_request_body: false)
       if skip_request_body
         body = "[Skipped]"
       else
         request_body = (request.body.respond_to?(:read) ? request.body.read : request.body)
-        body = request_body&.dup&.force_encoding("UTF-8")
-        begin
-          body = JSON.parse(body) if body.present?
-        rescue JSON::ParserError
-          body
-        end
+        body = parse_body(request_body)
       end
-      create(path: request.path, request_body: body, method: request.method, started_at: Time.current, loggable: loggable)
+      create(path: request.path, request_body: body, method: request.method, started_at: Time.current,
+        loggable: loggable, client_reference: client_reference)
+    end
+
+    # Bodies that are not valid UTF-8 (file uploads, images, PDFs...) can't be stored as text,
+    # so only their size is logged.
+    def self.parse_body(body)
+      return body unless body.is_a?(String)
+
+      body = body.dup.force_encoding("UTF-8")
+      return "[Binary: #{body.bytesize} bytes]" unless body.valid_encoding?
+      return body if body.blank?
+
+      JSON.parse(body)
+    rescue JSON::ParserError
+      body
     end
 
     def from_response(response, skip_response_body: false)
@@ -73,13 +78,7 @@ module RailsApiLogger
     private
 
     def manipulate_body(body)
-      body_duplicate = body&.dup&.force_encoding("UTF-8")
-      begin
-        body_duplicate = JSON.parse(body_duplicate) if body_duplicate.present?
-      rescue JSON::ParserError
-        body_duplicate
-      end
-      body_duplicate
+      self.class.parse_body(body)
     end
   end
 end

data/gemfiles/rails_8.0.gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: ..
   specs:
-    rails_api_logger (0.11.0)
-      activejob (>= 6.0)
-      activerecord (>= 6.0)
+    rails_api_logger (1.0.0)
+      activejob (>= 8.0)
+      activerecord (>= 8.0)
       nokogiri
-      railties (>= 6.0)
+      railties (>= 8.0)
       zeitwerk (>= 2.0.0)
 
 GEM
@@ -263,7 +263,7 @@ GEM
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
-    uri (1.0.3)
+    uri (1.1.2)
     useragent (0.16.11)
     webmock (3.25.1)
       addressable (>= 2.8.0)
@@ -295,4 +295,4 @@ DEPENDENCIES
   webmock (~> 3.25.0)
 
 BUNDLED WITH
-   2.5.4
+  4.0.20

data/gemfiles/{rails_7.1.gemfile → rails_8.1.gemfile}

@@ -2,10 +2,11 @@
 
 source "https://rubygems.org"
 
-gem "rails", "~> 7.1.0"
-gem "rspec-rails", "~> 7.1.0"
-gem "sqlite3", "~> 1.7.3"
+gem "rails", "~> 8.1.0"
+gem "rspec-rails", "~> 8.0"
+gem "sqlite3", "~> 2.1.0"
 gem "ammeter", "~> 1.1"
+gem "puma"
 gem "webmock", "~> 3.25.0"
 
 gemspec path: "../"