umbrellio-utils 1.14.2 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2630729cf41d11cf9483cba3097d1c5d949a0042f28a1b32e31e9fa1e227a6ce
-  data.tar.gz: 446cdaae850b3b8f7907cd02c3ed728df5380faec78e28c41875289a902bd0dd
+  metadata.gz: c27f256cd436b6d013e2e51538fd44b9fd8004a094f10c24d5bb169d932ddfbb
+  data.tar.gz: 68866be3591d51653d2099439bc9c1d75e627745ee75f2e7971313298c8b8a36
 SHA512:
-  metadata.gz: 9febf799edc6661b9f4ebf7187ed014a270371ae6cb0cd859199115038295ffeb65231e07ed26d5a6f17920272011e346075390c3f124a3ec99ccf69bfd81bd2
-  data.tar.gz: 0062de566624c25277163d639f62e8d1c8080a5ba0a18b3ea1304a296670758a0b92b7d721ab247ae1fcd9c91aef11f760642683ea12c43675bcc1dda3e09336
+  metadata.gz: 6043a41a4db19e94a620be02aeb9481f44eaab456b944f840fa185f467a6b74abdd58ed809c5505dd736c7d77caca990ea8e26f251552dd3227cbf448fd6612d
+  data.tar.gz: 7fc84ce2916828b0d67254ac6dcc25e92a9d9b05c6e865f27ef969a2bb7a6f7543056e48b726df522d7aa31a210a8c27f39198dfdb1bfa0b4531c07e81824a6a

data/.github/workflows/release.yml

@@ -2,41 +2,53 @@ name: Release
 
 on:
   push:
-    tags:
-      - "v*"
+    branches:
+      - main
 
 jobs:
-  gem:
-    runs-on: ubuntu-24.04
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v4
 
+      - name: Extract version
+        id: version
+        run: |
+          version=$(grep -oP '(?<=VERSION = ")[^"]+' lib/umbrellio_utils/version.rb)
+          echo "tag=v$version" >> "$GITHUB_OUTPUT"
+
+      - name: Check if release already exists
+        id: check
+        run: |
+          if gh release view "${{ steps.version.outputs.tag }}" &>/dev/null; then
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - uses: ruby/setup-ruby@v1
+        if: steps.check.outputs.exists == 'false'
         with:
           ruby-version: "3.4"
           bundler-cache: true
 
       - name: Build gem
-        run: bundle exec rake build
+        if: steps.check.outputs.exists == 'false'
+        run: gem build umbrellio_utils.gemspec
 
-      - name: Push to RubyGems
-        run: gem push pkg/*.gem
+      - name: Push gem to RubyGems
+        if: steps.check.outputs.exists == 'false'
+        run: gem push *.gem
         env:
           GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
 
-  github-release:
-    needs: gem
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Create GitHub release
-        uses: actions/create-release@v1
+      - uses: softprops/action-gh-release@v3
+        if: steps.check.outputs.exists == 'false'
         with:
-          tag_name: ${{ github.ref_name }}
-          release_name: ${{ github.ref_name }}
+          tag_name: ${{ steps.version.outputs.tag }}
+          files: "*.gem"
           generate_release_notes: true
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

data/Gemfile

@@ -11,6 +11,7 @@ gem "ci-helper"
 gem "click_house", github: "umbrellio/click_house", branch: "master"
 gem "clickhouse-native"
 gem "csv"
+gem "gvltools"
 gem "http"
 gem "net-pop"
 gem "nokogiri"

data/Gemfile.lock

@@ -11,7 +11,7 @@ GIT
 PATH
   remote: .
   specs:
-    umbrellio-utils (1.14.2)
+    umbrellio-utils (1.15.0)
       memery (~> 1)
 
 GEM
@@ -139,6 +139,7 @@ GEM
       net-http (~> 0.5)
     globalid (1.3.0)
       activesupport (>= 6.1)
+    gvltools (0.5.0)
     http (6.0.3)
       http-cookie (~> 1.0)
       llhttp (~> 0.6.1)
@@ -415,6 +416,7 @@ DEPENDENCIES
   click_house!
   clickhouse-native
   csv
+  gvltools
   http
   net-pop
   nokogiri

data/README.md

@@ -115,6 +115,32 @@ end
 Utils::Constants.useful_method #=> "Just string"
 ```
 
+### Instrumentation
+
+The gem ships a set of opt-in files for collecting GVL and allocation stats
+(they are not loaded by default and require the `gvltools` gem in your app):
+
+```ruby
+# Extends ActiveSupport::Notifications::Event with #gvl_time and #malloc_increase_bytes
+require "umbrellio_utils/patches/active_support_event"
+
+# Enriches the "Completed" action controller log entry (rails_semantic_logger)
+# with GC, GVL and allocation stats
+require "umbrellio_utils/patches/rails_semantic_logger"
+
+# Logs Sidekiq job completion with the same stats. Relies on the
+# "perform.sidekiq_job" notification published by the umbrellio fork of yabeda-sidekiq
+require "umbrellio_utils/semantic_logger/sidekiq_job_metrics"
+UmbrellioUtils::SemanticLogger::SidekiqJobMetrics.subscribe!
+```
+
+Don't forget to enable the GVL timer as early as possible (e.g. right after
+`Bundler.require` in `config/application.rb`):
+
+```ruby
+GVLTools::LocalTimer.enable
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/umbrellio/utils.

data/lib/umbrellio_utils/large_object.rb

@@ -41,9 +41,7 @@ module UmbrellioUtils
     end
 
     def delete!
-      run(:lo_unlink, oid)
-    rescue Sequel::DatabaseError
-      # Ignored
+      run(:lo_unlink, oid) if exists?
     end
 
     def exists?

data/lib/umbrellio_utils/patches/active_support_event.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "gvltools"
+require "active_support/notifications"
+
+module UmbrellioUtils
+  # Namespace for opt-in monkey-patches. None of these files are loaded by default,
+  # require them explicitly from your application.
+  module Patches
+    # Extends ActiveSupport::Notifications::Event with GVL wait time and malloc stats.
+    # https://github.com/rails/rails/blob/main/activesupport/lib/active_support/notifications/instrumenter.rb # rubocop:disable Layout/LineLength
+    #
+    # The GVL timer must be enabled in the application as early as possible:
+    #   GVLTools::LocalTimer.enable
+    module ActiveSupportEvent
+      STATS_PRECISION = 6
+
+      def initialize(...)
+        super
+        @gvl_time_start = 0
+        @gvl_time_finish = 0
+        @malloc_increase_bytes_start = 0
+        @malloc_increase_bytes_finish = 0
+      end
+
+      def start!
+        super
+        @gvl_time_start = GVLTools::LocalTimer.monotonic_time
+        @malloc_increase_bytes_start = GC.stat(:malloc_increase_bytes)
+      end
+
+      def finish!
+        super
+        @gvl_time_finish = GVLTools::LocalTimer.monotonic_time
+        @malloc_increase_bytes_finish = GC.stat(:malloc_increase_bytes)
+      end
+
+      # Time the thread spent waiting for the GVL, in milliseconds
+      def gvl_time
+        (@gvl_time_finish - @gvl_time_start) / 1_000_000.0
+      end
+
+      # Delta of the process-global malloc counter: can be negative (it resets on GC)
+      # and includes sibling threads' allocations under threaded servers. Guard with
+      # #positive? where a counter is expected.
+      def malloc_increase_bytes
+        @malloc_increase_bytes_finish - @malloc_increase_bytes_start
+      end
+
+      def stats(precision: STATS_PRECISION)
+        {
+          gc_time: gc_time.round(precision),
+          gvl_time: gvl_time.round(precision),
+          cpu_time: cpu_time.round(precision),
+          idle_time: idle_time.round(precision),
+          allocations:,
+          malloc_increase_bytes:,
+        }
+      end
+    end
+  end
+end
+
+ActiveSupport::Notifications::Event.prepend(UmbrellioUtils::Patches::ActiveSupportEvent)

data/lib/umbrellio_utils/patches/rails_semantic_logger.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative "active_support_event"
+
+module UmbrellioUtils
+  module Patches
+    # Simplifies the "Completed" action controller log entry and enriches it with
+    # GC, GVL and allocation stats.
+    # https://github.com/reidmorrison/rails_semantic_logger/blob/master/lib/rails_semantic_logger/action_controller/log_subscriber.rb # rubocop:disable Layout/LineLength
+    #
+    # Require this file after the rails_semantic_logger gem has been loaded
+    # (e.g. from an initializer).
+    module RailsSemanticLogger
+      PRECISION = 6
+      PARAMS_SIZE_LIMIT = 10_000
+
+      def process_action(event)
+        ::Rails.logger.info do
+          payload = event.payload.dup
+          payload[:path] = extract_path(payload[:path]) if payload.key?(:path)
+          payload[:params] = cap_params(payload[:params]) if payload.key?(:params)
+
+          payload[:view_time] = payload.delete(:view_runtime).to_f.round(PRECISION)
+          payload[:db_time] = payload.delete(:db_runtime).to_f.round(PRECISION)
+          payload.merge!(event.stats)
+
+          # Causes excessive log output with Rails 5 RC1
+          payload.delete(:headers)
+          # Causes recursion in Rails 6.1.rc1
+          payload.delete(:request)
+          payload.delete(:response)
+          # Keep only the [class, message] pair in :exception (set by ActiveSupport),
+          # not the raw exception with its backtrace — consistent with sidekiq_job_metrics.
+          payload.delete(:exception_object)
+
+          {
+            message: "Completed ##{payload[:action]}",
+            duration: event.duration,
+            payload:,
+          }
+        end
+      end
+
+      private
+
+      def cap_params(params)
+        serialized = params.to_json
+        return params if serialized.bytesize <= PARAMS_SIZE_LIMIT
+
+        {
+          truncated: true,
+          bytesize: serialized.bytesize,
+          preview: "#{serialized[0, PARAMS_SIZE_LIMIT]}...",
+        }
+      end
+    end
+  end
+end
+
+RailsSemanticLogger::ActionController::LogSubscriber.prepend(
+  UmbrellioUtils::Patches::RailsSemanticLogger,
+)

data/lib/umbrellio_utils/semantic_logger/sidekiq_job_metrics.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "../patches/active_support_event"
+
+module UmbrellioUtils
+  module SemanticLogger
+    # Logs Sidekiq job duration, GC, GVL and allocation stats as a dedicated line.
+    # Relies on the "perform.sidekiq_job" notification published by the umbrellio
+    # fork of yabeda-sidekiq. Call +subscribe!+ from an initializer.
+    module SidekiqJobMetrics
+      MESSAGE = "Sidekiq job stats"
+
+      extend self
+
+      def subscribe!
+        ActiveSupport::Notifications.subscribe("perform.sidekiq_job") do |event|
+          log_event(event)
+        end
+      end
+
+      private
+
+      def log_event(event)
+        logger = ::SemanticLogger[event.payload[:worker] || "Sidekiq"]
+        logger.info(
+          message: MESSAGE,
+          duration: event.duration,
+          payload: metrics_payload(event),
+        )
+      end
+
+      def metrics_payload(event)
+        payload = {
+          worker: event.payload[:worker],
+          queue: event.payload[:queue],
+          **event.stats,
+        }
+        # [class, message] pair set by ActiveSupport::Notifications when the job raised
+        payload[:exception] = event.payload[:exception] if event.payload[:exception]
+        payload
+      end
+    end
+  end
+end

data/lib/umbrellio_utils/semantic_logger/tiny_json_formatter.rb

@@ -20,6 +20,10 @@ module UmbrellioUtils
         time: :time,
       }.freeze
 
+      # Fields that are included in the output only when explicitly requested
+      # via +custom_names_mapping+ (e.g. <code>{ duration: :duration }</code>).
+      OPTIONAL_FIELDS = %i[duration payload].freeze
+
       # Returns a new instance of the {UmbrellioUtils::SemanticLogger::TinyJsonFormatter}.
       # @option [Integer] message_size_limit maximum number of characters in a log message
       # @option [Hash] custom_names_mapping mapping from default field names to custom ones.
@@ -31,6 +35,12 @@ module UmbrellioUtils
       # @option custom_names_mapping [Symbol] :tags custom name for the `tags` field.
       # @option custom_names_mapping [Symbol] :named_tags custom name for the `named_tags` field.
       # @option custom_names_mapping [Symbol] :time custom name for the `time` field.
+      # @option custom_names_mapping [Symbol] :duration enables logging of the `duration`
+      #   field under the given name. Unlike the default fields, it is not logged
+      #   unless explicitly listed in the mapping.
+      # @option custom_names_mapping [Symbol] :payload enables logging of the `payload`
+      #   field under the given name. Unlike the default fields, it is not logged
+      #   unless explicitly listed in the mapping.
       # @example Use custom name for the `message` and `time` fields
       #   UmbrellioUtils::SemanticLogger::TinyJsonFormatter.new(
       #     time: :timestamp, message: :note,
@@ -60,7 +70,17 @@ module UmbrellioUtils
       # Builds hash with data from log.
       # @return [Hash] the hash, which will be converted to the JSON later.
       def build_data_for(log)
-        field_names.values_at(*DEFAULT_NAMES_MAPPING.keys).zip(pack_data(log)).to_h
+        data = field_names.values_at(*DEFAULT_NAMES_MAPPING.keys).zip(pack_data(log)).to_h
+
+        OPTIONAL_FIELDS.each do |field|
+          field_name = field_names[field]
+          next if field_name.nil?
+
+          value = log.public_send(field)
+          data[field_name] = value unless value.nil?
+        end
+
+        data
       end
 
       # Builds an [Array] with all the required fields, which are arranged

data/lib/umbrellio_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UmbrellioUtils
-  VERSION = "1.14.2"
+  VERSION = "1.15.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: umbrellio-utils
 version: !ruby/object:Gem::Version
-  version: 1.14.2
+  version: 1.15.0
 platform: ruby
 authors:
 - Team Umbrellio
@@ -68,11 +68,14 @@ files:
 - lib/umbrellio_utils/misc.rb
 - lib/umbrellio_utils/parsing.rb
 - lib/umbrellio_utils/passwords.rb
+- lib/umbrellio_utils/patches/active_support_event.rb
+- lib/umbrellio_utils/patches/rails_semantic_logger.rb
 - lib/umbrellio_utils/random.rb
 - lib/umbrellio_utils/request_wrapper.rb
 - lib/umbrellio_utils/rounding.rb
 - lib/umbrellio_utils/rubocop.rb
 - lib/umbrellio_utils/rubocop/cop/custom/regex_rules.rb
+- lib/umbrellio_utils/semantic_logger/sidekiq_job_metrics.rb
 - lib/umbrellio_utils/semantic_logger/tiny_json_formatter.rb
 - lib/umbrellio_utils/sql.rb
 - lib/umbrellio_utils/store.rb