deprecation_collector 0.3.0 → 0.5.0

data/lib/deprecation_collector/web/views/show.html.template.rb

@@ -1,13 +1,82 @@
 _buf = ''; _buf << ("<header class=\"mb-3\"><h1>Deprecation</h1><a class=\"btn btn-secondary\"".freeze); 
 ; 
 ; 
-; _slim_codeattributes1 = deprecations_path; if _slim_codeattributes1; if _slim_codeattributes1 == true; _buf << (" href=\"\"".freeze); else; _buf << (" href=\"".freeze); _buf << ((::Temple::Utils.escape_html((_slim_codeattributes1))).to_s); _buf << ("\"".freeze); end; end; _buf << (">Back</a><a class=\"btn btn-danger\" data-confirm=\"Delete?\" data-method=\"delete\"".freeze); 
-; _slim_codeattributes2 = deprecation_path(@deprecation[:digest]); if _slim_codeattributes2; if _slim_codeattributes2 == true; _buf << (" href=\"\"".freeze); else; _buf << (" href=\"".freeze); _buf << ((::Temple::Utils.escape_html((_slim_codeattributes2))).to_s); _buf << ("\"".freeze); end; end; _buf << (" rel=\"nofollow\" title=\"Delete\"><span class=\"glyphicon glyphicon-trash insalesicon-trash\"></span>Delete</a></header><main><div class=\"card text-dark bg-light p-3\"><pre><code>".freeze); 
+; _slim_codeattributes1 = deprecations_path; if _slim_codeattributes1; if _slim_codeattributes1 == true; _buf << (" href=\"\"".freeze); else; _buf << (" href=\"".freeze); _buf << ((::Temple::Utils.escape_html((_slim_codeattributes1))).to_s); _buf << ("\"".freeze); end; end; _buf << (">Back</a> <a class=\"btn btn-danger\" data-confirm=\"Delete?\" data-method=\"delete\"".freeze); 
+; _slim_codeattributes2 = deprecation_path(@deprecation[:digest]); if _slim_codeattributes2; if _slim_codeattributes2 == true; _buf << (" href=\"\"".freeze); else; _buf << (" href=\"".freeze); _buf << ((::Temple::Utils.escape_html((_slim_codeattributes2))).to_s); _buf << ("\"".freeze); end; end; _buf << (" rel=\"nofollow\" title=\"Delete\"><span class=\"glyphicon glyphicon-trash insalesicon-trash\"></span>Delete</a> </header><main><dl class=\"row\"><dt>Message</dt><dd><code>".freeze); 
 ; 
 ; 
 ; 
 ; 
 ; 
 ; 
-; _buf << ((::Temple::Utils.escape_html((@deprecation.pretty_inspect))).to_s); 
-; _buf << ("</code></pre></div></main>".freeze); _buf
+; 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:message]))).to_s); 
+; _buf << ("</code></dd><dt>First time</dt><dd>".freeze); 
+; 
+; 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:first_timestamp]&.yield_self { |time| Time.at(time) }))).to_s); 
+; 
+; _buf << ("</dd>".freeze); if @deprecation[:count] > 1; 
+; _buf << ("<dt>Count</dt><dd>".freeze); 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:count]))).to_s); 
+; 
+; _buf << ("</dd>".freeze); end; _buf << ("<dt>Realm</dt><dd>".freeze); 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:realm]))).to_s); 
+; 
+; _buf << ("</dd>".freeze); if @deprecation[:ruby_version]; 
+; _buf << ("<dt>Ruby</dt><dd>".freeze); 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:ruby_version]))).to_s); 
+; _buf << ("</dd>".freeze); end; if @deprecation[:rails_version]; 
+; _buf << ("<dt>Rails</dt><dd>".freeze); 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:rails_version]))).to_s); 
+; 
+; _buf << ("</dd>".freeze); end; if @deprecation[:revision]; 
+; _buf << ("<dt>Revision</dt><dd>".freeze); 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:revision]))).to_s); 
+; _buf << ("</dd>".freeze); end; if @deprecation[:hostname]; 
+; _buf << ("<dt>Hostname</dt><dd>".freeze); 
+; _buf << ((::Temple::Utils.escape_html((@deprecation[:hostname]))).to_s); 
+; 
+; _buf << ("</dd>".freeze); end; if @deprecation[:context]; 
+; _buf << ("<dt>Context</dt><dd><div class=\"card p-3\"><pre><code>".freeze); 
+; 
+; 
+; 
+; _buf << ((::Temple::Utils.escape_html((JSON.pretty_generate(@deprecation[:context])))).to_s); 
+; 
+; _buf << ("</code></pre></div></dd>".freeze); end; if @deprecation[:app_traceline]; 
+; _buf << ("<dt>App traceline</dt><dd>".freeze); 
+; 
+; location, function = @deprecation[:app_traceline].split(':in `', 2); 
+; _buf << ("<code class=\"code_location\">".freeze); _buf << ((::Temple::Utils.escape_html((location))).to_s); 
+; _buf << ("</code> <i>".freeze); _buf << ((::Temple::Utils.escape_html((function.delete_suffix("'")))).to_s); 
+; 
+; _buf << ("</i></dd>".freeze); end; if @deprecation[:gem_traceline]; 
+; _buf << ("<dt>Gem traceline</dt><dd>".freeze); 
+; 
+; location, function = @deprecation[:gem_traceline].split(':in `', 2); 
+; full_gemname = location.delete_prefix('/gems/').gsub(%r{/.*}, ''); 
+; location_in_gem = location.delete_prefix("/gems/#{full_gemname}/"); 
+; _buf << ("<i>".freeze); _buf << ((::Temple::Utils.escape_html((full_gemname))).to_s); 
+; _buf << ("</i> <code class=\"code_location\"".freeze); _slim_codeattributes3 = location_in_gem; if _slim_codeattributes3; if _slim_codeattributes3 == true; _buf << (" data-copy-value=\"\"".freeze); else; _buf << (" data-copy-value=\"".freeze); _buf << ((::Temple::Utils.escape_html((_slim_codeattributes3))).to_s); _buf << ("\"".freeze); end; end; _buf << (">".freeze); _buf << ((::Temple::Utils.escape_html((location_in_gem.delete_prefix('lib/')))).to_s); 
+; _buf << ("</code> <i>".freeze); _buf << ((::Temple::Utils.escape_html((function.delete_suffix("'")))).to_s); 
+; 
+; _buf << ("</i></dd>".freeze); end; if @deprecation[:full_backtrace]; 
+; _buf << ("<dt>Backtrace</dt><dd>".freeze); 
+; 
+; @deprecation[:full_backtrace].each do |trace_line|; 
+; _buf << ("<div class=\"trace-line\">".freeze); 
+; location, function = trace_line.split(':in `', 2); 
+; _buf << ("<code class=\"code_location\">".freeze); _buf << ((::Temple::Utils.escape_html((location))).to_s); 
+; _buf << ("</code> <i>".freeze); _buf << ((::Temple::Utils.escape_html((function.delete_suffix("'")))).to_s); 
+; 
+; _buf << ("</i></div>".freeze); end; _buf << ("</dd>".freeze); end; fields_to_reject = %i[message first_timestamp count realm app_traceline gem_traceline full_backtrace ruby_version rails_version context revision hostname digest_base digest].to_set; 
+; additional_data = @deprecation.reject { |key,_val| fields_to_reject.include?(key) }; 
+; if additional_data.size > 0; 
+; _buf << ("<dt>Additional data <div class=\"card p-3\"><pre><code>".freeze); 
+; 
+; 
+; 
+; _buf << ((::Temple::Utils.escape_html((JSON.pretty_generate additional_data))).to_s); 
+; _buf << ("</code></pre></div></dt>".freeze); end; _buf << ("</dl><a".freeze); _slim_codeattributes4 = deprecation_path(params[:id], format: 'json'); if _slim_codeattributes4; if _slim_codeattributes4 == true; _buf << (" href=\"\"".freeze); else; _buf << (" href=\"".freeze); _buf << ((::Temple::Utils.escape_html((_slim_codeattributes4))).to_s); _buf << ("\"".freeze); end; end; _buf << (">Raw json</a></main>".freeze); 
+; _buf

data/lib/deprecation_collector/web.rb

@@ -4,10 +4,17 @@ require "erb"
 require "rack/content_length"
 require "rack/builder"
 
-require_relative 'web/application'
+require_relative "web/application"
 
 class DeprecationCollector
+  # rack app with a html interface to deprecation collector with a persistent storage like redis
   class Web
+    attr_accessor :import_enabled
+
+    def initialize(import_enabled: nil)
+      @import_enabled = import_enabled
+    end
+
     def self.call(env)
       @app ||= new
       @app.call(env)
@@ -22,11 +29,13 @@ class DeprecationCollector
     end
 
     private
+
     def build
+      web = self
       ::Rack::Builder.new do
         # use Rack::Static etc goes here
 
-        run Web::Application.new
+        run Web::Application.new(web)
       end
     end
   end

data/lib/deprecation_collector.rb

@@ -3,8 +3,8 @@
 require_relative "deprecation_collector/version"
 require_relative "deprecation_collector/deprecation"
 require_relative "deprecation_collector/collectors"
+require_relative "deprecation_collector/storage"
 require "time"
-require "redis"
 require "json"
 require "set"
 
@@ -52,37 +52,68 @@ class DeprecationCollector
   end
 
   # NB: count is expensive in production env (but possible if needed) - produces a lot of redis writes
-  attr_accessor :count, :raise_on_deprecation, :save_full_backtrace,
+  attr_accessor :raise_on_deprecation, :save_full_backtrace,
                 :exclude_realms,
-                :write_interval, :write_interval_jitter,
-                :app_revision, :app_root,
+                :app_name, :app_revision, :app_root,
                 :print_to_stderr, :print_recurring
-  attr_writer :redis, :context_saver, :fingerprinter
+  attr_reader :count, :write_interval, :write_interval_jitter, :key_prefix
+  attr_writer :context_saver, :fingerprinter
 
   def initialize(mutex: nil)
-    # on cruby hash itself is threadsafe, but we need to prevent races
-    @deprecations_mutex = mutex || Mutex.new
-    @deprecations = {}
-    @known_digests = Set.new
-    @last_write_time = current_time
     @enabled = true
+    @instance_mutex = mutex
 
     load_default_config
   end
 
   def load_default_config
-    @redis = defined?($redis) && $redis # rubocop:disable Style/GlobalVars
-    @count = false
+    if (redis = defined?($redis) && $redis) # rubocop:disable Style/GlobalVars
+      self.redis = redis
+    end
     @raise_on_deprecation = false
     @exclude_realms = []
     @ignore_message_regexp = nil
     @app_root = (defined?(Rails) && Rails.root.present? && Rails.root) || Dir.pwd
     # NB: in production with hugreds of workers may easily overload redis with writes, so more delay needed:
-    @write_interval = 900 # 15.minutes
-    @write_interval_jitter = 60
+    self.count = false
+    self.write_interval = 900 # 15.minutes
+    self.write_interval_jitter = 60
     @context_saver = nil
   end
 
+  def redis=(val)
+    raise ArgumentError, "redis should not be nil" unless val
+
+    @storage ||= DeprecationCollector::Storage::Redis.new(val, mutex: @instance_mutex, count: count, # rubocop:disable Naming/MemoizedInstanceVariableName
+                                                               write_interval: write_interval,
+                                                               write_interval_jitter: write_interval_jitter,
+                                                               key_prefix: key_prefix)
+  end
+
+  def count=(val)
+    storage.count = val if storage.respond_to?(:count)
+    @count = val
+  end
+
+  def write_interval=(val)
+    storage.write_interval = val if storage.respond_to?(:write_interval)
+    @write_interval = val
+  end
+
+  def write_interval_jitter=(val)
+    storage.write_interval_jitter = val if storage.respond_to?(:write_interval_jitter)
+    @write_interval_jitter = val
+  end
+
+  def key_prefix=(val)
+    storage.key_prefix = val
+    @key_prefix = val
+  end
+
+  def storage
+    @storage ||= DeprecationCollector::Storage::StdErr.new
+  end
+
   def ignored_messages=(val)
     @ignore_message_regexp = (val && Regexp.union(val)) || nil
   end
@@ -127,123 +158,81 @@ class DeprecationCollector
     @count
   end
 
-  def redis
-    raise "DeprecationCollector#redis is not set" unless @redis
-
-    @redis
-  end
-
   def write_to_redis(force: false)
-    return unless force || (@enabled && (current_time > @last_write_time + @write_interval))
-
-    deprecations_to_flush = nil
-    @deprecations_mutex.synchronize do
-      deprecations_to_flush = @deprecations
-      @deprecations = {}
-      @last_write_time = current_time
-      # checking in this section to prevent multiple parallel check requests
-      return (@enabled = false) unless enabled_in_redis?
-    end
+    return unless force || @enabled
 
-    write_count_to_redis(deprecations_to_flush) if count?
-
-    # make as few writes as possible, other workers may already have reported our warning
-    fetch_known_digests
-    deprecations_to_flush.reject! { |digest, _val| @known_digests.include?(digest) }
-    return unless deprecations_to_flush.any?
-
-    @known_digests.merge(deprecations_to_flush.keys)
-    @redis.mapped_hmset("deprecations:data", deprecations_to_flush.transform_values(&:to_json))
+    storage.flush(force: force)
   end
 
   # prevent fresh process from wiring frequent already known messages
   def fetch_known_digests
-    @known_digests.merge(@redis.hkeys("deprecations:data"))
+    storage.fetch_known_digests
   end
 
   def flush_redis(enable: false)
-    @redis.del("deprecations:data", "deprecations:counter", "deprecations:notes")
-    @redis.del("deprecations:enabled") if enable
-    @deprecations.clear
-    @known_digests.clear
+    storage.clear(enable: enable)
   end
 
   def enabled_in_redis?
-    @redis.get("deprecations:enabled") != "false"
+    storage.enabled?
+  end
+
+  def enabled?
+    @enabled
   end
 
   def enable
+    storage.enable
     @enabled = true
-    @redis.set("deprecations:enabled", "true")
   end
 
   def disable
     @enabled = false
-    @redis.set("deprecations:enabled", "false")
+    storage.disable
   end
 
   def dump
-    read_each.to_a.to_json
+    read_each.to_a.compact.to_json
+  end
+
+  def import_dump(json)
+    dump = JSON.parse(json)
+    # TODO: some checks
+
+    digests = dump.map { |dep| dep["digest"] }
+    raise "need digests" unless digests.none?(&:nil?)
+
+    dump_hash = dump.map { |dep| [dep.delete("digest"), dep] }.to_h
+
+    storage.import(dump_hash)
   end
 
   def read_each
     return to_enum(:read_each) unless block_given?
 
-    cursor = 0
-    loop do
-      cursor, data_pairs = @redis.hscan("deprecations:data", cursor)
-
-      if data_pairs.any?
-        data_pairs.zip(
-          @redis.hmget("deprecations:counter", data_pairs.map(&:first)),
-          @redis.hmget("deprecations:notes", data_pairs.map(&:first))
-        ).each do |(digest, data), count, notes|
-          yield decode_deprecation(digest, data, count, notes)
-        end
-      end
-      break if cursor == "0"
+    storage.read_each do |digest, data, count, notes|
+      yield decode_deprecation(digest, data, count, notes)
     end
   end
 
   def read_one(digest)
-    decode_deprecation(
-      digest,
-      *@redis.pipelined do |pipe|
-        pipe.hget("deprecations:data", digest)
-        pipe.hget("deprecations:counter", digest)
-        pipe.hget("deprecations:notes", digest)
-      end
-    )
+    decode_deprecation(*storage.read_one(digest))
   end
 
   def delete_deprecations(remove_digests)
-    return 0 unless remove_digests.any?
-
-    @redis.pipelined do |pipe|
-      pipe.hdel("deprecations:data", *remove_digests)
-      pipe.hdel("deprecations:notes", *remove_digests)
-      pipe.hdel("deprecations:counter", *remove_digests) if @count
-    end.first
-  end
-
-  def cleanup
-    cursor = 0
-    removed = total = 0
-    loop do
-      cursor, data_pairs = @redis.hscan("deprecations:data", cursor) # NB: some pages may be empty
-      total += data_pairs.size
-      removed += delete_deprecations(
-        data_pairs.to_h.select { |_digest, data| !block_given? || yield(JSON.parse(data, symbolize_names: true)) }.keys
-      )
-      break if cursor == "0"
-    end
-    "#{removed} removed, #{total - removed} left"
+    storage.delete(remove_digests)
+  end
+
+  def cleanup(&block)
+    raise ArgumentError, "provide a block to filter deprecations" unless block
+
+    storage.cleanup(&block)
   end
 
   protected
 
   def unsent_deprecations
-    @deprecations
+    storage.unsent_deprecations
   end
 
   def store_deprecation(deprecation, allow_context: true)
@@ -251,51 +240,34 @@ class DeprecationCollector
 
     deprecation.context = context_saver.call if context_saver && allow_context
     deprecation.custom_fingerprint = fingerprinter.call(deprecation) if fingerprinter && allow_context
+    deprecation.app_name = app_name if app_name
 
-    fresh = !@deprecations.key?(deprecation.digest)
-    @deprecations_mutex.synchronize do
-      (@deprecations[deprecation.digest] ||= deprecation).touch
-    end
-
-    write_to_redis if current_time - @last_write_time > (@write_interval + rand(@write_interval_jitter))
-    fresh
+    storage.store(deprecation)
   end
 
   def log_deprecation_if_needed(deprecation, fresh)
     return unless print_to_stderr && !deprecation.ignored?
     return unless fresh || print_recurring
 
+    log_deprecation(deprecation)
+  end
+
+  def log_deprecation(deprecation)
     msg = deprecation.message
     msg = "DEPRECATION: #{msg}" unless msg.start_with?("DEPRECAT")
     $stderr.puts(msg) # rubocop:disable Style/StderrPuts
   end
 
-  def current_time
-    return Time.zone.now if Time.respond_to?(:zone) && Time.zone
-
-    Time.now
-  end
-
   def decode_deprecation(digest, data, count, notes)
     return nil unless data
 
     data = JSON.parse(data, symbolize_names: true)
-    unless data.is_a?(Hash)
-      # this should not happen (this means broken Deprecation#to_json or some data curruption)
-      return nil
-    end
+    # this should not happen (this means broken Deprecation#to_json or some data curruption)
+    return nil unless data.is_a?(Hash)
 
     data[:digest] = digest
     data[:notes] = JSON.parse(notes, symbolize_names: true) if notes
     data[:count] = count.to_i if count
     data
   end
-
-  def write_count_to_redis(deprecations_to_flush)
-    @redis.pipelined do |pipe|
-      deprecations_to_flush.each_pair do |digest, deprecation|
-        pipe.hincrby("deprecations:counter", digest, deprecation.occurences)
-      end
-    end
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deprecation_collector
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Vasily Fedoseyev
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-26 00:00:00.000000000 Z
+date: 2023-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Collects and aggregates warnings and deprecations. Optimized for production
   environment.
 email:
@@ -66,12 +80,14 @@ files:
 - lib/deprecation_collector.rb
 - lib/deprecation_collector/collectors.rb
 - lib/deprecation_collector/deprecation.rb
+- lib/deprecation_collector/storage.rb
 - lib/deprecation_collector/version.rb
 - lib/deprecation_collector/web.rb
 - lib/deprecation_collector/web/application.rb
 - lib/deprecation_collector/web/helpers.rb
 - lib/deprecation_collector/web/router.rb
 - lib/deprecation_collector/web/utils.rb
+- lib/deprecation_collector/web/views/import.html.template.rb
 - lib/deprecation_collector/web/views/index.html.template.rb
 - lib/deprecation_collector/web/views/layout.html.template.rb
 - lib/deprecation_collector/web/views/show.html.template.rb
@@ -98,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Collector for ruby/rails deprecations and warnings, suitable for production