deprecation_collector 0.4.0 → 0.5.0

data/lib/deprecation_collector.rb

@@ -3,8 +3,8 @@
 require_relative "deprecation_collector/version"
 require_relative "deprecation_collector/deprecation"
 require_relative "deprecation_collector/collectors"
+require_relative "deprecation_collector/storage"
 require "time"
-require "redis"
 require "json"
 require "set"
 
@@ -52,37 +52,68 @@ class DeprecationCollector
   end
 
   # NB: count is expensive in production env (but possible if needed) - produces a lot of redis writes
-  attr_accessor :count, :raise_on_deprecation, :save_full_backtrace,
+  attr_accessor :raise_on_deprecation, :save_full_backtrace,
                 :exclude_realms,
-                :write_interval, :write_interval_jitter,
-                :app_revision, :app_root,
+                :app_name, :app_revision, :app_root,
                 :print_to_stderr, :print_recurring
-  attr_writer :redis, :context_saver, :fingerprinter
+  attr_reader :count, :write_interval, :write_interval_jitter, :key_prefix
+  attr_writer :context_saver, :fingerprinter
 
   def initialize(mutex: nil)
-    # on cruby hash itself is threadsafe, but we need to prevent races
-    @deprecations_mutex = mutex || Mutex.new
-    @deprecations = {}
-    @known_digests = Set.new
-    @last_write_time = current_time
     @enabled = true
+    @instance_mutex = mutex
 
     load_default_config
   end
 
   def load_default_config
-    @redis = defined?($redis) && $redis # rubocop:disable Style/GlobalVars
-    @count = false
+    if (redis = defined?($redis) && $redis) # rubocop:disable Style/GlobalVars
+      self.redis = redis
+    end
     @raise_on_deprecation = false
     @exclude_realms = []
     @ignore_message_regexp = nil
     @app_root = (defined?(Rails) && Rails.root.present? && Rails.root) || Dir.pwd
     # NB: in production with hugreds of workers may easily overload redis with writes, so more delay needed:
-    @write_interval = 900 # 15.minutes
-    @write_interval_jitter = 60
+    self.count = false
+    self.write_interval = 900 # 15.minutes
+    self.write_interval_jitter = 60
     @context_saver = nil
   end
 
+  def redis=(val)
+    raise ArgumentError, "redis should not be nil" unless val
+
+    @storage ||= DeprecationCollector::Storage::Redis.new(val, mutex: @instance_mutex, count: count, # rubocop:disable Naming/MemoizedInstanceVariableName
+                                                               write_interval: write_interval,
+                                                               write_interval_jitter: write_interval_jitter,
+                                                               key_prefix: key_prefix)
+  end
+
+  def count=(val)
+    storage.count = val if storage.respond_to?(:count)
+    @count = val
+  end
+
+  def write_interval=(val)
+    storage.write_interval = val if storage.respond_to?(:write_interval)
+    @write_interval = val
+  end
+
+  def write_interval_jitter=(val)
+    storage.write_interval_jitter = val if storage.respond_to?(:write_interval_jitter)
+    @write_interval_jitter = val
+  end
+
+  def key_prefix=(val)
+    storage.key_prefix = val
+    @key_prefix = val
+  end
+
+  def storage
+    @storage ||= DeprecationCollector::Storage::StdErr.new
+  end
+
   def ignored_messages=(val)
     @ignore_message_regexp = (val && Regexp.union(val)) || nil
   end
@@ -127,59 +158,37 @@ class DeprecationCollector
     @count
   end
 
-  def redis
-    raise "DeprecationCollector#redis is not set" unless @redis
-
-    @redis
-  end
-
   def write_to_redis(force: false)
-    return unless force || (@enabled && (current_time > @last_write_time + @write_interval))
-
-    deprecations_to_flush = nil
-    @deprecations_mutex.synchronize do
-      deprecations_to_flush = @deprecations
-      @deprecations = {}
-      @last_write_time = current_time
-      # checking in this section to prevent multiple parallel check requests
-      return (@enabled = false) unless enabled_in_redis?
-    end
-
-    write_count_to_redis(deprecations_to_flush) if count?
-
-    # make as few writes as possible, other workers may already have reported our warning
-    fetch_known_digests
-    deprecations_to_flush.reject! { |digest, _val| @known_digests.include?(digest) }
-    return unless deprecations_to_flush.any?
+    return unless force || @enabled
 
-    @known_digests.merge(deprecations_to_flush.keys)
-    @redis.mapped_hmset("deprecations:data", deprecations_to_flush.transform_values(&:to_json))
+    storage.flush(force: force)
   end
 
   # prevent fresh process from wiring frequent already known messages
   def fetch_known_digests
-    @known_digests.merge(@redis.hkeys("deprecations:data"))
+    storage.fetch_known_digests
   end
 
   def flush_redis(enable: false)
-    @redis.del("deprecations:data", "deprecations:counter", "deprecations:notes")
-    @redis.del("deprecations:enabled") if enable
-    @deprecations.clear
-    @known_digests.clear
+    storage.clear(enable: enable)
   end
 
   def enabled_in_redis?
-    @redis.get("deprecations:enabled") != "false"
+    storage.enabled?
+  end
+
+  def enabled?
+    @enabled
   end
 
   def enable
+    storage.enable
     @enabled = true
-    @redis.set("deprecations:enabled", "true")
   end
 
   def disable
     @enabled = false
-    @redis.set("deprecations:enabled", "false")
+    storage.disable
   end
 
   def dump
@@ -191,71 +200,39 @@ class DeprecationCollector
     # TODO: some checks
 
     digests = dump.map { |dep| dep["digest"] }
-    raise 'need digests' unless digests.none?(&:nil?)
+    raise "need digests" unless digests.none?(&:nil?)
 
-    dump_hash = dump.map { |dep| [dep.delete('digest'), dep] }.to_h
+    dump_hash = dump.map { |dep| [dep.delete("digest"), dep] }.to_h
 
-    @redis.mapped_hmset("deprecations:data", dump_hash.transform_values(&:to_json))
+    storage.import(dump_hash)
   end
 
   def read_each
     return to_enum(:read_each) unless block_given?
 
-    cursor = 0
-    loop do
-      cursor, data_pairs = @redis.hscan("deprecations:data", cursor)
-
-      if data_pairs.any?
-        data_pairs.zip(
-          @redis.hmget("deprecations:counter", data_pairs.map(&:first)),
-          @redis.hmget("deprecations:notes", data_pairs.map(&:first))
-        ).each do |(digest, data), count, notes|
-          yield decode_deprecation(digest, data, count, notes)
-        end
-      end
-      break if cursor == "0"
+    storage.read_each do |digest, data, count, notes|
+      yield decode_deprecation(digest, data, count, notes)
     end
   end
 
   def read_one(digest)
-    decode_deprecation(
-      digest,
-      *@redis.pipelined do |pipe|
-        pipe.hget("deprecations:data", digest)
-        pipe.hget("deprecations:counter", digest)
-        pipe.hget("deprecations:notes", digest)
-      end
-    )
+    decode_deprecation(*storage.read_one(digest))
   end
 
   def delete_deprecations(remove_digests)
-    return 0 unless remove_digests.any?
-
-    @redis.pipelined do |pipe|
-      pipe.hdel("deprecations:data", *remove_digests)
-      pipe.hdel("deprecations:notes", *remove_digests)
-      pipe.hdel("deprecations:counter", *remove_digests) if @count
-    end.first
-  end
-
-  def cleanup
-    cursor = 0
-    removed = total = 0
-    loop do
-      cursor, data_pairs = @redis.hscan("deprecations:data", cursor) # NB: some pages may be empty
-      total += data_pairs.size
-      removed += delete_deprecations(
-        data_pairs.to_h.select { |_digest, data| !block_given? || yield(JSON.parse(data, symbolize_names: true)) }.keys
-      )
-      break if cursor == "0"
-    end
-    "#{removed} removed, #{total - removed} left"
+    storage.delete(remove_digests)
+  end
+
+  def cleanup(&block)
+    raise ArgumentError, "provide a block to filter deprecations" unless block
+
+    storage.cleanup(&block)
   end
 
   protected
 
   def unsent_deprecations
-    @deprecations
+    storage.unsent_deprecations
   end
 
   def store_deprecation(deprecation, allow_context: true)
@@ -263,51 +240,34 @@ class DeprecationCollector
 
     deprecation.context = context_saver.call if context_saver && allow_context
     deprecation.custom_fingerprint = fingerprinter.call(deprecation) if fingerprinter && allow_context
+    deprecation.app_name = app_name if app_name
 
-    fresh = !@deprecations.key?(deprecation.digest)
-    @deprecations_mutex.synchronize do
-      (@deprecations[deprecation.digest] ||= deprecation).touch
-    end
-
-    write_to_redis if current_time - @last_write_time > (@write_interval + rand(@write_interval_jitter))
-    fresh
+    storage.store(deprecation)
   end
 
   def log_deprecation_if_needed(deprecation, fresh)
     return unless print_to_stderr && !deprecation.ignored?
     return unless fresh || print_recurring
 
+    log_deprecation(deprecation)
+  end
+
+  def log_deprecation(deprecation)
     msg = deprecation.message
     msg = "DEPRECATION: #{msg}" unless msg.start_with?("DEPRECAT")
     $stderr.puts(msg) # rubocop:disable Style/StderrPuts
   end
 
-  def current_time
-    return Time.zone.now if Time.respond_to?(:zone) && Time.zone
-
-    Time.now
-  end
-
   def decode_deprecation(digest, data, count, notes)
     return nil unless data
 
     data = JSON.parse(data, symbolize_names: true)
-    unless data.is_a?(Hash)
-      # this should not happen (this means broken Deprecation#to_json or some data curruption)
-      return nil
-    end
+    # this should not happen (this means broken Deprecation#to_json or some data curruption)
+    return nil unless data.is_a?(Hash)
 
     data[:digest] = digest
     data[:notes] = JSON.parse(notes, symbolize_names: true) if notes
     data[:count] = count.to_i if count
     data
   end
-
-  def write_count_to_redis(deprecations_to_flush)
-    @redis.pipelined do |pipe|
-      deprecations_to_flush.each_pair do |digest, deprecation|
-        pipe.hincrby("deprecations:counter", digest, deprecation.occurences)
-      end
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deprecation_collector
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Vasily Fedoseyev
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-28 00:00:00.000000000 Z
+date: 2023-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -80,6 +80,7 @@ files:
 - lib/deprecation_collector.rb
 - lib/deprecation_collector/collectors.rb
 - lib/deprecation_collector/deprecation.rb
+- lib/deprecation_collector/storage.rb
 - lib/deprecation_collector/version.rb
 - lib/deprecation_collector/web.rb
 - lib/deprecation_collector/web/application.rb
@@ -98,7 +99,7 @@ metadata:
   homepage_uri: https://github.com/Vasfed/deprecation_collector
   source_code_uri: https://github.com/Vasfed/deprecation_collector
   changelog_uri: https://github.com/Vasfed/deprecation_collector/blob/main/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -113,8 +114,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: Collector for ruby/rails deprecations and warnings, suitable for production
 test_files: []