deprecation_collector 0.0.1 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7936a3e78d550c3940f0abe364171df3b39cc1e19931cfba23b8cfdb060122ca
-  data.tar.gz: 2a4d06a0baee48c61c027c7791fef7894b5af5d1791fe0e3c1bd3afaf2fa61ab
+  metadata.gz: 2342f445e6c89ba4ea74094dd39894d99ad6e546781405fe43a6ecba71396264
+  data.tar.gz: d5d6edaa942f57a726f9336a06541dabc3eadae2731d01953443a88d84fc7c2e
 SHA512:
-  metadata.gz: c5a10c07173fe8b73003630c2fa0839b07c62f0471b72269aabdcece19935f6693fc6b0a3a3c8634b1731b7cc9db0aae186ff2067a43047692e15f32180d5210
-  data.tar.gz: b6e598c579eb46756b9a78336fd3b223d4224b01f5a677b55375732279564c9b609dc2e87083a6cb47cb5e86c567c73102465080e275818564d94c1d29b29408
+  metadata.gz: 9ac2e881d1687dc3756904840f21f98cc62dc497cac31804fcfeac8e4e01c780b5370497ed52da2d1ec22ca14a68ee3be4dc163c5d48cd47c38b775ef0df7b1c
+  data.tar.gz: 5725ea9137d79a83ea8d1df1ae9b458fe0f94479d5e7c9643ac15368ef853f75fac38cce3b7e8543fd4d03df884fa3999019d08902ba437a196f6b5b3c78dd06

data/.rubocop.yml

@@ -1,5 +1,7 @@
 AllCops:
-  TargetRubyVersion: 2.6
+  # spec/ have TargetRubyVersion: 2.6
+  TargetRubyVersion: 2.4
+  NewCops: enable
 
 Style/StringLiterals:
   Enabled: true
@@ -11,3 +13,8 @@ Style/StringLiteralsInInterpolation:
 
 Layout/LineLength:
   Max: 120
+
+Metrics/ClassLength: { Max: 200 }
+Metrics/MethodLength: { Max: 15 }
+Metrics/CyclomaticComplexity: { Max: 9 }
+Metrics/PerceivedComplexity: { Max: 9 }

data/CHANGELOG.md

@@ -1,5 +1,13 @@
+== 0.0.4
+- added first_timestamp to deprecations (unix timestamp of first occurrence, not accurate because a worker with later timestamp may dump its deprecations earlier)
 
+== 0.0.3
+- Fixed selective deprecation cleanup (`DeprecationCollector.instance.cleanup { |d| d[:message].include?('foo') }`)
+
+== 0.0.2
+
+- Reorganized code
 
 == 0.0.1
 
-Initial release
+- Initial release

data/Gemfile

@@ -11,6 +11,10 @@ gem "rspec", "~> 3.0"
 gem "timecop"
 
 gem "rubocop", "~> 1.21"
+gem "rubocop-performance"
+gem "rubocop-rails"
+gem "rubocop-rake"
+gem "rubocop-rspec"
 
 # TODO: appraisals
-gem "rails", '6.0'
+gem "rails", "6.0"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deprecation_collector (0.0.1)
+    deprecation_collector (0.0.4)
       redis (>= 2.0)
 
 GEM
@@ -123,7 +123,7 @@ GEM
       thor (>= 0.20.3, < 2.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    redis (4.6.0)
+    redis (4.7.1)
     regexp_parser (2.5.0)
     rexml (3.2.5)
     rspec (3.11.0)
@@ -150,6 +150,17 @@ GEM
       unicode-display_width (>= 1.4.0, < 3.0)
     rubocop-ast (1.18.0)
       parser (>= 3.1.1.0)
+    rubocop-performance (1.14.3)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rails (2.15.2)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 1.7.0, < 2.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.11.1)
+      rubocop (~> 1.19)
     ruby-progressbar (1.11.0)
     sprockets (4.0.3)
       concurrent-ruby (~> 1.0)
@@ -178,6 +189,10 @@ DEPENDENCIES
   rake (~> 13.0)
   rspec (~> 3.0)
   rubocop (~> 1.21)
+  rubocop-performance
+  rubocop-rails
+  rubocop-rake
+  rubocop-rspec
   timecop
 
 BUNDLED WITH

data/README.md

@@ -1,21 +1,39 @@
 # DeprecationCollector
+[![Gem Version](https://badge.fury.io/rb/deprecation_collector.svg)](https://badge.fury.io/rb/deprecation_collector)
 
 Collects ruby and rails deprecation warnings.
+Designed to be suitable for use in production under load.
+
 (gem is a work-in-process, documentation will come later)
 
 ## Installation
 
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add deprecation_collector
-
-If bundler is not being used to manage dependencies, install the gem by executing:
-
-    $ gem install deprecation_collector
+```sh
+bundle add deprecation_collector
+```
 
 ## Usage
 
-
+Add an initializer with configuration, like
+
+```ruby
+  Rails.application.config.to_prepare do
+    DeprecationCollector.install do |instance|
+      instance.redis = Redis.new # default is $redis
+      instance.app_revision = ::GIT_REVISION
+      instance.count = false
+      instance.save_full_backtrace = true
+      instance.raise_on_deprecation = false
+      instance.write_interval = (::Rails.env.production? && 15.minutes) || 1.minute
+      instance.exclude_realms = %i[kernel] if Rails.env.production?
+      instance.ignored_messages = [
+        "Ignoring db/schema_cache.yml because it has expired"
+      ]
+    end
+  end
+```
 
 ## Development
 
@@ -25,7 +43,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/deprecation_collector.
+Bug reports and pull requests are welcome on GitHub at https://github.com/Vasfed/deprecation_collector.
 
 ## License
 

data/lib/deprecation_collector/collectors.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+# :nodoc:
+class DeprecationCollector
+  class << self
+    protected
+
+    def install_collectors
+      tap_activesupport if defined?(ActiveSupport::Deprecation)
+      tap_kernel
+      tap_warning_class
+    end
+
+    def tap_warning_class
+      Warning.singleton_class.prepend(DeprecationCollector::WarningCollector)
+      Warning[:deprecated] = true if Warning.respond_to?(:[]=) # turn on ruby 2.7 deprecations
+    end
+
+    def tap_activesupport
+      # TODO: a more polite hook
+      ActiveSupport::Deprecation.behavior = lambda do |message, callstack, deprecation_horizon, gem_name|
+        # not polite to turn off all other possible behaviors, but otherwise may get duplicate calls
+        DeprecationCollector.collect(message, callstack, :rails)
+        ActiveSupport::Deprecation::DEFAULT_BEHAVIORS[stock_activesupport_behavior].call(
+          message, callstack, deprecation_horizon, gem_name
+        )
+      end
+    end
+
+    def stock_activesupport_behavior
+      Rails.application&.config&.active_support&.deprecation || :log
+    end
+
+    def tap_kernel
+      Kernel.class_eval do
+        # module is included in others thus prepend does not work
+        remove_method :warn
+        class << self
+          remove_method :warn
+        end
+        module_function(define_method(:warn) do |*messages, **kwargs|
+          KernelWarningCollector.warn(*messages, backtrace: caller, **kwargs)
+        end)
+      end
+    end
+  end
+
+  # Ruby sometimes has two warnings for one actual occurence
+  # Example:
+  # caller.rb:1: warning: Passing the keyword argument as the last hash parameter is deprecated
+  # calleee.rb:1: warning: The called method `method_name' is defined here
+  module MultipartWarningJoiner
+    module_function
+
+    def two_part_warning?(str)
+      # see ruby src - `rb_warn`, `rb_compile_warn`
+      str.end_with?(
+        "uses the deprecated method signature, which takes one parameter\n", # respond_to?
+        # 2.7 kwargs:
+        "maybe ** should be added to the call\n",
+        "Passing the keyword argument as the last hash parameter is deprecated\n", # бывает и не двойной
+        "Splitting the last argument into positional and keyword parameters is deprecated\n"
+      ) ||
+        str.include?("warning: already initialized constant") ||
+        str.include?("warning: method redefined; discarding old")
+    end
+
+    def handle(new_str)
+      old_str = Thread.current[:multipart_warning_str]
+      Thread.current[:multipart_warning_str] = nil
+      if old_str
+        return yield(old_str + new_str) if new_str.include?("is defined here") || new_str.include?(" was here")
+
+        yield(old_str)
+      end
+
+      return (Thread.current[:multipart_warning_str] = new_str) if two_part_warning?(new_str)
+
+      yield(new_str)
+    end
+  end
+
+  # taps into ruby core Warning#warn
+  module WarningCollector
+    def warn(str)
+      backtrace = caller
+      MultipartWarningJoiner.handle(str) do |multi_str|
+        DeprecationCollector.collect(multi_str, backtrace, :warning)
+      end
+    end
+  end
+
+  # for tapping into Kernel#warn
+  module KernelWarningCollector
+    module_function
+
+    def warn(*messages, backtrace: nil, **_kwargs)
+      backtrace ||= caller
+      str = messages.map(&:to_s).join("\n").strip
+      DeprecationCollector.collect(str, backtrace, :kernel)
+      # not passing to `super` - it will pass to Warning#warn, we do not want that
+    end
+  end
+end

data/lib/deprecation_collector/deprecation.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+class DeprecationCollector
+  # :nodoc:
+  class Deprecation
+    attr_reader :message, :realm, :gem_traceline, :app_traceline, :occurences, :first_timestamp, :full_backtrace
+
+    CLEANUP_REGEXES = {
+      # rails views generated methods names are unique per-worker
+      /_app_views_(\w+)__(\d+)_(\d+)/ => "_app_views_\\1__",
+
+      # repl line numbers are not important, may be ignore all repl at all
+      /\A\((pry|irb)\):\d+/ => '(\1)'
+    }.freeze
+
+    def initialize(message, realm = nil, backtrace = [], cleanup_prefixes = [])
+      # backtrace is Thread::Backtrace::Location or array of strings for other realms
+      @message = message.dup
+      @realm = realm
+      @occurences = 0
+      @gem_traceline = find_gem_traceline(backtrace)
+      @app_traceline = find_app_traceline(backtrace)
+      @first_timestamp = Time.now.to_i
+
+      cleanup_prefixes.each do |path|
+        @gem_traceline.delete_prefix!(path)
+        @message.gsub!(path, "")
+      end
+
+      CLEANUP_REGEXES.each_pair do |regex, replace|
+        @gem_traceline&.gsub!(regex, replace)
+        @app_traceline&.gsub!(regex, replace)
+      end
+
+      @full_backtrace = backtrace.map(&:to_s) if DeprecationCollector.instance.save_full_backtrace
+    end
+
+    def touch
+      @occurences += 1
+    end
+
+    def ignored?
+      false
+    end
+
+    def message_for_digest
+      # some gems like rest-client put data in warnings, need to aggregate
+      # + some bactrace per-worker unique method names may be there
+      @message.gsub(/"(?:[^"\\]|\\.)*"/, '""').gsub(/__\d+_\d+/, "___").gsub(/\((pry|irb)\):\d+/, '(\1)')
+    end
+
+    def digest
+      @digest ||= Digest::MD5.hexdigest(digest_base)
+    end
+
+    def digest_base
+      "1:#{RUBY_VERSION}:#{Rails.version}:#{message_for_digest}:#{gem_traceline}:#{app_traceline}"
+    end
+
+    def as_json(_options = {})
+      {
+        message: message,
+        realm: realm,
+        app_traceline: app_traceline,
+        gem_traceline: (gem_traceline != app_traceline && gem_traceline) || nil,
+        full_backtrace: full_backtrace,
+        ruby_version: RUBY_VERSION,
+        rails_version: (defined?(Rails) && Rails.version),
+        hostname: Socket.gethostname,
+        revision: DeprecationCollector.instance.app_revision,
+        count: @occurences, # output anyway for frequency estimation (during write_interval inside single process)
+        first_timestamp: first_timestamp, # this may not be accurate, a worker with later timestamp may dump earlier
+        digest_base: digest_base # for debug purposes
+      }.compact
+    end
+
+    protected
+
+    def find_app_traceline(backtrace)
+      app_root = DeprecationCollector.instance.app_root_prefix
+      backtrace.find do |line|
+        line = line.to_s
+        (!line.start_with?("/") || line.start_with?(app_root)) && !line.include?("/gems/")
+      end&.to_s&.dup&.delete_prefix(app_root)
+    end
+
+    def find_gem_traceline(backtrace)
+      backtrace.find { |line| !line.to_s.include?("kernel_warn") }&.to_s&.dup || backtrace.first.to_s.dup
+    end
+  end
+end

data/lib/deprecation_collector/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class DeprecationCollector
-  VERSION = "0.0.1"
+  VERSION = "0.0.4"
 end

data/lib/deprecation_collector.rb

@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 require_relative "deprecation_collector/version"
-require 'time'
-require 'redis'
+require_relative "deprecation_collector/deprecation"
+require_relative "deprecation_collector/collectors"
+require "time"
+require "redis"
+require "json"
 
+# singleton class for collector
 class DeprecationCollector
-  # NB: in production with hugreds of workers may easily overload redis with writes, so more delay needed:
-  FLUSH_INTERVAL = (::Rails.env.production? && 15.minutes) || 1.minute
-
   @instance_mutex = Mutex.new
   @installed = false
   private_class_method :new
@@ -15,9 +16,13 @@ class DeprecationCollector
   def self.instance
     return @instance if defined?(@instance) && @instance
 
+    create_instance
+  end
+
+  def self.create_instance
     @instance_mutex.synchronize do
-      # переиспользовать мутекс не обязательно, но он используется ровно один раз
-      @instance ||= new($redis, mutex: @instance_mutex)
+      # no real need to reuse the mutex, but it is used only once here anyway
+      @instance ||= new(mutex: @instance_mutex)
     end
     @instance
   end
@@ -28,7 +33,7 @@ class DeprecationCollector
 
   # inside dev env may be called multiple times
   def self.install
-    instance # to make it created, configuration comes later
+    create_instance # to make it created, configuration comes later
 
     @instance_mutex.synchronize do
       unless @installed
@@ -37,213 +42,63 @@ class DeprecationCollector
       end
 
       yield instance if block_given?
+      instance.fetch_known_digests
 
-      # TODO: a more polite hook
-      ActiveSupport::Deprecation.behavior = lambda do |message, callstack, deprecation_horizon, gem_name|
-        # not polite to turn off all other possible behaviors, but otherwise may get duplicate calls
-        DeprecationCollector.collect(message, callstack, :rails)
-        ActiveSupport::Deprecation::DEFAULT_BEHAVIORS[Rails.application&.config&.active_support&.deprecation || :log].call(
-          message, callstack, deprecation_horizon, gem_name
-        )
-      end
-
-      Kernel.class_eval do
-        # module is included in others thus prepend does not work
-        remove_method :warn
-        class << self
-          remove_method :warn
-        end
-        module_function(define_method(:warn) do |*messages, **kwargs|
-          KernelWarningCollector.warn(*messages, backtrace: caller, **kwargs)
-        end)
-      end
-
-      Warning.singleton_class.prepend(DeprecationCollector::WarningCollector)
-      Warning[:deprecated] = true if Warning.respond_to?(:[]=) # turn on ruby 2.7 deprecations
+      install_collectors
     end
 
     @instance
   end
 
-  module MultipartWarningJoiner
-    module_function
-
-    # Ruby sometimes has two warnings for one actual occurence
-    # Example:
-    # caller.rb:1: warning: Passing the keyword argument as the last hash parameter is deprecated
-    # calleee.rb:1: warning: The called method `method_name' is defined here
-    def two_part_warning?(str)
-      # see ruby src - `rb_warn`, `rb_compile_warn`
-      str.end_with?(
-        "uses the deprecated method signature, which takes one parameter\n", # respond_to?
-        # 2.7 kwargs:
-        "maybe ** should be added to the call\n",
-        "Passing the keyword argument as the last hash parameter is deprecated\n", # бывает и не двойной
-        "Splitting the last argument into positional and keyword parameters is deprecated\n"
-      ) ||
-        str.include?("warning: already initialized constant") ||
-        str.include?("warning: method redefined; discarding old")
-    end
-
-    def handle(new_str)
-      old_str = Thread.current[:multipart_warning_str]
-      Thread.current[:multipart_warning_str] = nil
-      if old_str
-        return yield(old_str + new_str) if new_str.include?('is defined here') || new_str.include?(' was here')
-        yield(old_str)
-      end
-
-      if two_part_warning?(new_str)
-        Thread.current[:multipart_warning_str] = new_str
-        return
-      end
-
-      yield(new_str)
-    end
-  end
-
-  # taps into ruby core Warning#warn
-  module WarningCollector
-    def warn(str)
-      backtrace = caller
-      MultipartWarningJoiner.handle(str) do |multi_str|
-        DeprecationCollector.collect(multi_str, backtrace, :warning)
-      end
-    end
-  end
-
-  module KernelWarningCollector
-    module_function
-
-    def warn(*messages, backtrace: nil, **_kwargs)
-      backtrace ||= caller
-      str = messages.map(&:to_s).join("\n").strip
-      DeprecationCollector.collect(str, backtrace, :kernel)
-      # not passing to `super` - it will pass to Warning#warn, we do not want that
-    end
-  end
-
-  class Deprecation
-    attr_reader :message, :realm, :gem_traceline, :app_traceline, :occurences, :full_backtrace
-
-    def initialize(message, realm = nil, backtrace = [], cleanup_prefixes = [])
-      # backtrace is Thread::Backtrace::Location or array of strings for other realms
-      @message = message.dup
-      @realm = realm
-      @occurences = 0
-      @gem_traceline = backtrace.find { |line| !line.to_s.include?('kernel_warn') }&.to_s&.dup ||
-                       backtrace.first.to_s.dup
-      cleanup_prefixes.each do |path|
-        @gem_traceline.delete_prefix!(path)
-        @message.gsub!(path, '')
-      end
-
-      app_root = "#{DeprecationCollector.instance.app_root}/" # rubocop:disable Rails/FilePath
-      @app_traceline = backtrace.find do |line|
-        line = line.to_s
-        (!line.start_with?('/') || line.start_with?(app_root)) && !line.include?('/gems/')
-      end&.to_s&.dup&.delete_prefix(app_root)
-
-      # rails views generated methods names are unique per-worker
-      @gem_traceline&.gsub!(/_app_views_(\w+)__(\d+)_(\d+)/, "_app_views_\\1__")
-      @app_traceline&.gsub!(/_app_views_(\w+)__(\d+)_(\d+)/, "_app_views_\\1__")
-
-      # repl line numbers are not important, may be ignore all repl at all
-      @app_traceline&.gsub!(/\A\((pry|irb)\):\d+/, '(\1)')
-      @gem_traceline&.gsub!(/\A\((pry|irb)\):\d+/, '(\1)') # may contain app traceline, so filter too
-
-      @full_backtrace = backtrace.map(&:to_s) if DeprecationCollector.instance.save_full_backtrace
-    end
-
-    def touch
-      @occurences += 1
-    end
-
-    def ignored?
-      false
-    end
-
-    def message_for_digest
-      # some gems like rest-client put data in warnings, need to aggregate
-      # + some bactrace per-worker unique method names may be there
-      @message.gsub(/"(?:[^"\\]|\\.)*"/, '""').gsub(/__\d+_\d+/, '___').gsub(/\((pry|irb)\):\d+/, '(\1)')
-    end
-
-    def digest
-      @digest ||= Digest::MD5.hexdigest(digest_base)
-    end
-
-    def digest_base
-      "1:#{RUBY_VERSION}:#{Rails.version}:#{message_for_digest}:#{gem_traceline}:#{app_traceline}"
-    end
-
-    def as_json(_options = {})
-      {
-        message: message,
-        realm: realm,
-        app_traceline: app_traceline,
-        gem_traceline: (gem_traceline != app_traceline && gem_traceline) || nil,
-        full_backtrace: full_backtrace,
-        ruby_version: RUBY_VERSION,
-        rails_version: Rails.version,
-        hostname: Socket.gethostname,
-        revision: DeprecationCollector.instance.app_revision,
-        count: @occurences, # output anyway for frequency estimation (during write_interval inside single process)
-        digest_base: digest_base # for debug purposes
-      }.compact
-    end
-  end
-
+  # NB: count is expensive in production env (but possible if needed) - produces a lot of redis writes
   attr_accessor :count, :raise_on_deprecation, :save_full_backtrace,
                 :exclude_realms,
                 :write_interval, :write_interval_jitter,
                 :app_revision, :app_root
+  attr_writer :redis
 
-  def initialize(redis, mutex: nil)
-    @redis = redis
+  def initialize(mutex: nil)
     # on cruby hash itself is threadsafe, but we need to prevent races
     @deprecations_mutex = mutex || Mutex.new
     @deprecations = {}
     @known_digests = Set.new
     @last_write_time = current_time
+    @enabled = true
 
-    # default config:
+    load_default_config
+  end
+
+  def load_default_config
+    @redis = defined?($redis) && $redis # rubocop:disable Style/GlobalVars
     @count = false
     @raise_on_deprecation = false
     @exclude_realms = []
-    @write_interval = FLUSH_INTERVAL
-    @write_interval_jitter = 60
-    @enabled = true
     @ignore_message_regexp = nil
-    @app_root = defined?(Rails) && Rails.root.present? || Dir.pwd
-
-    fetch_known_digests # prevent fresh process from wiring frequent already known messages
+    @app_root = (defined?(Rails) && Rails.root.present? && Rails.root) || Dir.pwd
+    # NB: in production with hugreds of workers may easily overload redis with writes, so more delay needed:
+    @write_interval = 900 # 15.minutes
+    @write_interval_jitter = 60
   end
 
   def ignored_messages=(val)
     @ignore_message_regexp = (val && Regexp.union(val)) || nil
   end
 
+  def app_root_prefix
+    "#{app_root}/"
+  end
+
   def cleanup_prefixes
-    @cleanup_prefixes ||= Gem.path + ["#{app_root}/"] # rubocop:disable Rails/FilePath
+    @cleanup_prefixes ||= Gem.path + [app_root_prefix]
   end
 
   def collect(message, backtrace, realm = :unknown)
-    return unless @enabled
-    return if exclude_realms.include?(realm)
-    return if @ignore_message_regexp&.match?(message)
+    return if !@enabled || exclude_realms.include?(realm) || @ignore_message_regexp&.match?(message)
     raise "Deprecation: #{message}" if @raise_on_deprecation
 
     deprecation = Deprecation.new(message, realm, backtrace, cleanup_prefixes)
-    return if deprecation.ignored?
-
-    @deprecations_mutex.synchronize do
-      (@deprecations[deprecation.digest] ||= deprecation).touch
-    end
-
-    write_to_redis if current_time - @last_write_time > (@write_interval + rand(@write_interval_jitter))
-
-    $stderr.puts(message) if Rails.env.development? # rubocop:disable Style/StderrPuts
+    store_deprecation(deprecation)
+    log_deprecation_if_needed(deprecation)
   end
 
   def unsent_data?
@@ -254,40 +109,27 @@ class DeprecationCollector
     @count
   end
 
-  def fetch_known_digests
-    @known_digests.merge(@redis.hkeys('deprecations:data'))
+  def redis
+    raise "DeprecationCollector#redis is not set" unless @redis
+
+    @redis
   end
 
-  def write_to_redis(force: false)
-    return unless @enabled || force
+  def write_to_redis(force: false) # rubocop:disable Metrics/AbcSize
+    return unless force || (@enabled && (current_time > @last_write_time + @write_interval))
 
     deprecations_to_flush = nil
     @deprecations_mutex.synchronize do
-      # check in this section to prevent multiple check requests
-      unless enabled_in_redis?
-        @enabled = false
-        @deprecations = {}
-        return
-      end
-
-      return unless force || current_time > @last_write_time + @write_interval
-
       deprecations_to_flush = @deprecations
       @deprecations = {}
       @last_write_time = current_time
+      # checking in this section to prevent multiple parallel check requests
+      return (@enabled = false) unless enabled_in_redis?
     end
 
-    # count is expensive in production env (but possible if needed) - a lot of redis writes
-    if count?
-      @redis.pipelined do
-        deprecations_to_flush.each_pair do |digest, deprecation|
-          @redis.hincrby("deprecations:counter", digest, deprecation.occurences)
-        end
-      end
-    end
+    write_count_to_redis(deprecations_to_flush) if count?
 
     # make as few writes as possible, other workers may already have reported our warning
-    # TODO: at some point turn off writes?
     fetch_known_digests
     deprecations_to_flush.reject! { |digest, _val| @known_digests.include?(digest) }
     return unless deprecations_to_flush.any?
@@ -296,6 +138,11 @@ class DeprecationCollector
     @redis.mapped_hmset("deprecations:data", deprecations_to_flush.transform_values(&:to_json))
   end
 
+  # prevent fresh process from wiring frequent already known messages
+  def fetch_known_digests
+    @known_digests.merge(@redis.hkeys("deprecations:data"))
+  end
+
   def flush_redis(enable: false)
     @redis.del("deprecations:data", "deprecations:counter", "deprecations:notes")
     @redis.del("deprecations:enabled") if enable
@@ -317,6 +164,10 @@ class DeprecationCollector
     @redis.set("deprecations:enabled", "false")
   end
 
+  def dump
+    read_each.to_a.to_json
+  end
+
   def read_each
     return to_enum(:read_each) unless block_given?
 
@@ -339,49 +190,59 @@ class DeprecationCollector
   def read_one(digest)
     decode_deprecation(
       digest,
-      @redis.hget("deprecations:data", digest),
-      @redis.hget("deprecations:counter", digest),
-      @redis.hget("deprecations:notes", digest)
+      *@redis.pipelined do
+        @redis.hget("deprecations:data", digest)
+        @redis.hget("deprecations:counter", digest)
+        @redis.hget("deprecations:notes", digest)
+      end
     )
   end
 
   def delete_deprecations(remove_digests)
+    return 0 unless remove_digests.any?
+
     @redis.pipelined do
       @redis.hdel("deprecations:data", *remove_digests)
       @redis.hdel("deprecations:notes", *remove_digests)
       @redis.hdel("deprecations:counter", *remove_digests) if @count
-    end
+    end.first
   end
 
   def cleanup
     cursor = 0
-    removed = 0
-    total = 0
+    removed = total = 0
     loop do
-      cursor, data_pairs = @redis.hscan("deprecations:data", cursor)
-
-      if data_pairs.any?
-        remove_digests = []
-        total += data_pairs.size
-        data_pairs.each do |(digest, data)|
-          data = JSON.parse(data, symbolize_names: true)
-          remove_digests << digest if !block_given? || yield(data)
-        end
-
-        if remove_digests.any?
-          delete_deprecations(remove_digests)
-          removed += remove_digests.size
-        end
-      end
+      cursor, data_pairs = @redis.hscan("deprecations:data", cursor) # NB: some pages may be empty
+      total += data_pairs.size
+      removed += delete_deprecations(
+        data_pairs.to_h.select { |_digest, data| !block_given? || yield(JSON.parse(data, symbolize_names: true)) }.keys
+      )
       break if cursor == "0"
     end
     "#{removed} removed, #{total - removed} left"
   end
 
-  private
+  protected
+
+  def store_deprecation(deprecation)
+    return if deprecation.ignored?
+
+    @deprecations_mutex.synchronize do
+      (@deprecations[deprecation.digest] ||= deprecation).touch
+    end
+
+    write_to_redis if current_time - @last_write_time > (@write_interval + rand(@write_interval_jitter))
+  end
+
+  def log_deprecation_if_needed(deprecation)
+    return unless defined?(Rails) && Rails.env.development? && !deprecation.ignored?
+
+    $stderr.puts(deprecation.message) # rubocop:disable Style/StderrPuts
+  end
 
   def current_time
     return Time.zone.now if Time.respond_to?(:zone) && Time.zone
+
     Time.now
   end
 
@@ -392,4 +253,12 @@ class DeprecationCollector
     data[:count] = count.to_i if count
     data
   end
+
+  def write_count_to_redis(deprecations_to_flush)
+    @redis.pipelined do
+      deprecations_to_flush.each_pair do |digest, deprecation|
+        @redis.hincrby("deprecations:counter", digest, deprecation.occurences)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deprecation_collector
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.4
 platform: ruby
 authors:
 - Vasily Fedoseyev
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-29 00:00:00.000000000 Z
+date: 2022-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -43,6 +43,8 @@ files:
 - Rakefile
 - deprecation_collector.gemspec
 - lib/deprecation_collector.rb
+- lib/deprecation_collector/collectors.rb
+- lib/deprecation_collector/deprecation.rb
 - lib/deprecation_collector/version.rb
 - sig/deprecation_collector.rbs
 homepage: https://github.com/Vasfed/deprecation_collector