deprecation_collector 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7eda71f5ba3b7d7c1aa9a1c622a29e4329b8ab08bccc53d243aabea3a80a70cb
-  data.tar.gz: 213fa97a6ceede04c3f37160defb2a6732d84a32f25681a9486a567eae6d7fed
+  metadata.gz: 7f62a11c42b2d8d6ee4236c98960b452d5aa2c0ec76d83d6215101de55edf6a8
+  data.tar.gz: aff231737c13f8e47c63e60706cbee894d50da276caff00ec4898c766da9ee19
 SHA512:
-  metadata.gz: d4b4cf6feb4d736114f7324a0b34a65e32ee7f3edd610ee5aabb41daa069fd2417a746e927761c28ec22b37a9ceb1f768c2fa92064c11e0d4e4474801dd9f008
-  data.tar.gz: 9e6641fc8b17e5d4d2a5d5d7981afecca83feffbc549d88206105a4f752feeb951290fa9dc77f8d24166519df12da917d83f81685eb2d79b6ca87912d09ff599
+  metadata.gz: d053f33b41ca1dd5b20b81069bef9cf178f4f10fe7a00f193d8a5bdb4e1aa30267223df74bfe3abd234dd08ba2ccfaef757b5febbe8546393c808c1b7b68ecc7
+  data.tar.gz: b4c259673e8b2d4b19de8e3f259563ad19e34381c43b1dfb141e9193bc1f57606eb2b48c81dfb88b699440bbf4f5922a2af92c4ead1c066a90e468bb82a973c0

data/.rubocop.yml

@@ -10,6 +10,7 @@ AllCops:
   SuggestExtensions: false
   Exclude:
     - gemfiles/*
+    - lib/deprecation_collector/web/views/*.template.rb
 
 Style/StringLiterals:
   Enabled: true
@@ -30,3 +31,4 @@ Metrics/PerceivedComplexity: { Max: 9 }
 
 RSpec/ExampleLength: { Enabled: false }
 RSpec/MultipleExpectations: { Enabled: false }
+RSpec/MessageSpies: { Enabled: false }

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+== 0.5.0
+- more work on ui
+- refactored to separate deprecations storage from other logic
+- when redis is not provided - print all messages to stderr
+- added `key_prefix` option (default `'deprecations'`, location may change in the future) to allow multiple independent apps to write to one redis
+- added `app_name` option to record app name as separate field
+
 == 0.4.0
 - a bit better ui
 - simple import/export

data/Gemfile

@@ -20,14 +20,14 @@ unless defined?(Appraisal)
   end
 
   gem "rails", "~>6.0.0"
-  gem 'simplecov'
+  gem "simplecov"
 end
 
 gem "fakeredis"
 gem "redis", "~>4.8"
 
 # for web tests
-gem 'rack'
-gem 'webrick'
+gem "rack"
+gem "webrick"
 
-gem 'slim' # not used in production, for compiling templates
+gem "slim" # not used in production, for compiling templates

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deprecation_collector (0.4.0)
+    deprecation_collector (0.5.0)
       redis (>= 3.0)
 
 GEM

data/Rakefile

@@ -8,16 +8,17 @@ RSpec::Core::RakeTask.new(:spec)
 begin
   require "rubocop/rake_task"
   RuboCop::RakeTask.new
-rescue LoadError
+rescue LoadError # rubocop:disable Lint/SuppressedException
 end
 
 task default: %i[spec rubocop]
 
+desc "Compile slim templates (so that slim is not needed as dependency)"
 task :precompile_templates do
-  require 'slim'
+  require "slim"
   # Slim::Template.new { '.lala' }.precompiled_template
-  Dir['lib/deprecation_collector/web/views/*.slim'].each do |file|
-    target = file.sub(/\.slim\z/, '.template.rb')
+  Dir["lib/deprecation_collector/web/views/*.slim"].each do |file|
+    target = file.sub(/\.slim\z/, ".template.rb")
     puts "Compiling #{file} -> #{target}"
     content = Slim::Template.new(file).precompiled_template # maybe send(:precompiled, []) is more correct
 

data/deprecation_collector.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
       (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)}) ||
         f.match(%r{\Alib/deprecation_collector/web/views/.+\.slim\z})
     end
-  end + Dir['lib/deprecation_collector/web/views/*.slim'].map { |template| template.sub(/\.slim\z/, '.template.rb') }
+  end + Dir["lib/deprecation_collector/web/views/*.slim"].map { |template| template.sub(/\.slim\z/, ".template.rb") }
   spec.require_paths = ["lib"]
 
   spec.add_dependency "redis", ">= 3.0"

data/gemfiles/rails_6.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    deprecation_collector (0.4.0)
+    deprecation_collector (0.5.0)
       redis (>= 3.0)
 
 GEM

data/gemfiles/rails_7.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    deprecation_collector (0.4.0)
+    deprecation_collector (0.5.0)
       redis (>= 3.0)
 
 GEM

data/gemfiles/rails_none.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    deprecation_collector (0.4.0)
+    deprecation_collector (0.5.0)
       redis (>= 3.0)
 
 GEM

data/lib/deprecation_collector/deprecation.rb

@@ -4,7 +4,7 @@ class DeprecationCollector
   # :nodoc:
   class Deprecation
     attr_reader :message, :realm, :gem_traceline, :app_traceline, :occurences, :first_timestamp, :full_backtrace
-    attr_accessor :context, :custom_fingerprint
+    attr_accessor :context, :custom_fingerprint, :app_name
 
     CLEANUP_REGEXES = {
       # rails views generated methods names are unique per-worker
@@ -61,6 +61,7 @@ class DeprecationCollector
 
     def as_json(_options = {})
       {
+        app: app_name,
         message: message,
         realm: realm,
         app_traceline: app_traceline,

data/lib/deprecation_collector/storage.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require "redis"
+
+class DeprecationCollector
+  module Storage
+    # :nodoc:
+    class Base
+      # rubocop:disable Style/SingleLineMethods
+      def enabled?; true; end
+      def enable; end
+      def disable; end
+
+      def unsent_deprecations; []; end
+      def fetch_known_digests; end
+
+      def delete(digests); end
+      def clear(enable: false); end
+
+      def store(_deprecation); raise("Not implemented"); end
+      # rubocop:enable Style/SingleLineMethods
+    end
+
+    # dummy strategy that outputs every deprecation into stderr
+    class StdErr < Base
+      def store(deprecation)
+        DeprecationCollector.instance.send(:log_deprecation, deprecation)
+      end
+    end
+
+    # storing in redis with deduplication by fingerprint
+    class Redis < Base
+      attr_accessor :write_interval, :write_interval_jitter, :redis, :count
+
+      def initialize(redis, mutex: nil, count: false, write_interval: 900, write_interval_jitter: 60,
+                            key_prefix: nil)
+        super()
+        @key_prefix = key_prefix || "deprecations"
+        @redis = redis
+        @last_write_time = current_time
+        @count = count
+        @write_interval = write_interval
+        @write_interval_jitter = write_interval_jitter
+        # on cruby hash itself is threadsafe, but we need to prevent races
+        @deprecations_mutex = mutex || Mutex.new
+        @deprecations = {}
+        @known_digests = Set.new
+      end
+
+      def unsent_deprecations
+        @deprecations
+      end
+
+      def enabled?
+        @redis.get(enabled_flag_key) != "false"
+      end
+
+      def enable
+        @redis.set(enabled_flag_key, "true")
+      end
+
+      def disable
+        @redis.set(enabled_flag_key, "false")
+      end
+
+      def delete(remove_digests)
+        return 0 unless remove_digests.any?
+
+        @redis.pipelined do |pipe|
+          pipe.hdel(data_hash_key, *remove_digests)
+          pipe.hdel(notes_hash_key, *remove_digests)
+          pipe.hdel(counter_hash_key, *remove_digests) if @count
+        end.first
+      end
+
+      def clear(enable: false)
+        @redis.del(data_hash_key, counter_hash_key, notes_hash_key)
+        @redis.del(enabled_flag_key) if enable
+        @known_digests.clear
+        @deprecations.clear
+      end
+
+      def fetch_known_digests
+        # FIXME: use `.merge!`?
+        @known_digests.merge(@redis.hkeys(data_hash_key))
+      end
+
+      def store(deprecation)
+        fresh = !@deprecations.key?(deprecation.digest)
+        @deprecations_mutex.synchronize do
+          (@deprecations[deprecation.digest] ||= deprecation).touch
+        end
+
+        flush if current_time - @last_write_time > (@write_interval + rand(@write_interval_jitter))
+        fresh
+      end
+
+      def flush(force: false)
+        return unless force || (current_time > @last_write_time + @write_interval)
+
+        deprecations_to_flush = nil
+        @deprecations_mutex.synchronize do
+          deprecations_to_flush = @deprecations
+          @deprecations = {}
+          @last_write_time = current_time
+          # checking in this section to prevent multiple parallel check requests
+          return DeprecationCollector.instance.instance_variable_set(:@enabled, false) unless enabled?
+        end
+
+        write_count_to_redis(deprecations_to_flush) if @count
+
+        # make as few writes as possible, other workers may already have reported our warning
+        fetch_known_digests
+        deprecations_to_flush.reject! { |digest, _val| @known_digests.include?(digest) }
+        return unless deprecations_to_flush.any?
+
+        @known_digests.merge(deprecations_to_flush.keys)
+        @redis.mapped_hmset(data_hash_key, deprecations_to_flush.transform_values(&:to_json))
+      end
+
+      def read_each
+        cursor = 0
+        loop do
+          cursor, data_pairs = @redis.hscan(data_hash_key, cursor)
+
+          if data_pairs.any?
+            data_pairs.zip(
+              @redis.hmget(counter_hash_key, data_pairs.map(&:first)),
+              @redis.hmget(notes_hash_key, data_pairs.map(&:first))
+            ).each do |(digest, data), count, notes|
+              yield(digest, data, count, notes)
+            end
+          end
+          break if cursor == "0"
+        end
+      end
+
+      def read_one(digest)
+        [
+          digest,
+          *@redis.pipelined do |pipe|
+            pipe.hget(data_hash_key, digest)
+            pipe.hget(counter_hash_key, digest)
+            pipe.hget(notes_hash_key, digest)
+          end
+        ]
+      end
+
+      def import(dump_hash)
+        @redis.mapped_hmset(data_hash_key, dump_hash.transform_values(&:to_json))
+      end
+
+      def cleanup(&_block)
+        cursor = 0
+        removed = total = 0
+        loop do
+          cursor, data_pairs = @redis.hscan(data_hash_key, cursor) # NB: some pages may be empty
+          total += data_pairs.size
+          removed += delete(
+            data_pairs.to_h.select { |_digest, data| yield(JSON.parse(data, symbolize_names: true)) }.keys
+          )
+          break if cursor == "0"
+        end
+        "#{removed} removed, #{total - removed} left"
+      end
+
+      def key_prefix=(val)
+        @enabled_flag_key = @data_hash_key = @counter_hash_key = @notes_hash_key = nil
+        @key_prefix = val
+      end
+
+      protected
+
+      def enabled_flag_key
+        @enabled_flag_key ||= "#{@key_prefix}:enabled" # usually deprecations:enabled
+      end
+
+      def data_hash_key
+        @data_hash_key ||= "#{@key_prefix}:data" # usually deprecations:data
+      end
+
+      def counter_hash_key
+        @counter_hash_key ||= "#{@key_prefix}:counter" # usually deprecations:counter
+      end
+
+      def notes_hash_key
+        @notes_hash_key ||= "#{@key_prefix}:notes" # usually deprecations:notes
+      end
+
+      def current_time
+        return Time.zone.now if Time.respond_to?(:zone) && Time.zone
+
+        Time.now
+      end
+
+      def write_count_to_redis(deprecations_to_flush)
+        @redis.pipelined do |pipe|
+          deprecations_to_flush.each_pair do |digest, deprecation|
+            pipe.hincrby(counter_hash_key, digest, deprecation.occurences)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deprecation_collector/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class DeprecationCollector
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/deprecation_collector/web/application.rb

@@ -1,22 +1,21 @@
 # frozen_string_literal: true
 
-require_relative 'router'
-require_relative 'helpers'
-require 'pp'
+require_relative "router"
+require_relative "helpers"
 
 class DeprecationCollector
   class Web
+    # :nodoc:
     class Application
       extend Web::Router
       helpers Helpers
 
       attr_reader :web
+
       def initialize(web)
         @web = web
-        unless defined?(Temple::Utils) || ENV['DEPRECATION_COLLECTOR_RELOAD_WEB_TEMPLATES']
-          # used for escaping in compiled slim templates
-          require_relative 'utils'
-        end
+        # used for escaping in compiled slim templates
+        require_relative "utils" unless defined?(Temple::Utils) || ENV["DEPRECATION_COLLECTOR_RELOAD_WEB_TEMPLATES"]
       end
 
       def call(env)
@@ -25,7 +24,7 @@ class DeprecationCollector
 
       root do # index
         @deprecations = collector_instance.read_each.to_a.compact
-        @deprecations = @deprecations.sort_by { |dep| dep[:message] } unless params[:sort] == '0'
+        @deprecations = @deprecations.sort_by { |dep| dep[:message] } unless params[:sort] == "0"
 
         if params[:reject]
           @deprecations = @deprecations.reject { |dep| dep[:message].match?(Regexp.union(Array(params[:reject]))) }
@@ -38,50 +37,59 @@ class DeprecationCollector
         render slim: "index.html"
       end
 
-      get '/dump.json' do
+      get "/dump.json" do
         render json: collector_instance.dump
       end
 
-      get '/import' do
+      get "/import" do
         return "Import not enabled" unless import_enabled?
 
         render slim: "import.html"
       end
 
-      post '/import' do
-        halt 422, "need multipart json file" unless env['CONTENT_TYPE']&.start_with?('multipart/form-data') && params.dig(:file, :tempfile)
+      post "/import" do
+        unless env["CONTENT_TYPE"]&.start_with?("multipart/form-data") && params.dig(:file, :tempfile)
+          halt 422, "need multipart json file"
+        end
         collector_instance.import_dump(File.read(params[:file][:tempfile]))
         redirect_to deprecations_path
       end
 
-      get '/:id' do # show
+      get "/:id.json" do # show
+        @deprecation = collector_instance.read_one(params[:id])
+        halt 404 unless @deprecation
+        render json: JSON.pretty_generate(@deprecation)
+      end
+
+      get "/:id" do # show
         @deprecation = collector_instance.read_one(params[:id])
+        halt 404 unless @deprecation
         render slim: "show.html"
       end
 
-      delete '/all' do
+      delete "/all" do
         collector_instance.flush_redis
         redirect_to deprecations_path
       end
 
-      post '/enable' do
+      post "/enable" do
         collector_instance.enable
         redirect_to deprecations_path
       end
 
-      delete '/disable' do
+      delete "/disable" do
         collector_instance.disable
         redirect_to deprecations_path
       end
 
       # NB: order for wildcards is important
-      delete '/:id' do # destroy
+      delete "/:id" do # destroy
         collector_instance.delete_deprecations([params[:id]])
         redirect_to deprecations_path
       end
 
-      post '/trigger' do # trigger
-        trigger_kwargs_error_warning({ foo: nil }) if RUBY_VERSION.start_with?('2.7')
+      post "/trigger" do # trigger
+        trigger_kwargs_error_warning({ foo: nil }) if RUBY_VERSION.start_with?("2.7")
         trigger_rails_deprecation
         collector_instance.collect(
           "TestFoo#assign_attributes called (test attr_spy) trigger_rails_deprecation", caller_locations, :attr_spy

data/lib/deprecation_collector/web/helpers.rb

@@ -2,6 +2,7 @@
 
 class DeprecationCollector
   class Web
+    # :nodoc:
     module Helpers
       def collector_instance
         @collector_instance || DeprecationCollector.instance
@@ -17,15 +18,15 @@ class DeprecationCollector
       end
 
       def current_path
-        @current_path ||= request.path_info.gsub(/^\//, "")
+        @current_path ||= request.path_info.gsub(%r{^/}, "")
       end
 
       def deprecations_path
-        "#{root_path}"
+        root_path # /
       end
 
-      def deprecation_path(id)
-        "#{root_path}#{id}"
+      def deprecation_path(id, format: nil)
+        ["#{root_path}#{id}", format].compact.join('.')
       end
 
       def enable_deprecations_path
@@ -48,27 +49,37 @@ class DeprecationCollector
 
       def trigger_rails_deprecation
         return unless defined?(ActiveSupport::Deprecation)
+
         -> { ActiveSupport::Deprecation.warn("Test deprecation") } []
       end
 
       def current_color_theme
-        return 'dark' if params['dark']
-        return 'light' if params['light']
-        return 'dark' if request.get_header('HTTP_Sec_CH_Prefers_Color_Scheme').to_s.downcase.include?("dark")
-        'auto'
+        return "dark" if params["dark"]
+        return "light" if params["light"]
+        return "dark" if request.get_header("HTTP_Sec_CH_Prefers_Color_Scheme").to_s.downcase.include?("dark")
+
+        "auto"
+      end
+
+      def detect_tag(deprecation)
+        msg = deprecation[:message]
+        return :kwargs if msg.include?("Using the last argument as keyword parameters is deprecated") ||
+                          msg.include?("Passing the keyword argument as the last hash parameter is deprecated")
+        return :test if msg.include?("trigger_kwargs_error_warning") || msg.include?("trigger_rails_deprecation")
       end
 
       def deprecation_tags(deprecation)
-        {}.tap do |tags|
-          tags[:kwargs] = 'bg-secondary' if deprecation[:message].include?("Using the last argument as keyword parameters is deprecated") ||
-                                            deprecation[:message].include?("Passing the keyword argument as the last hash parameter is deprecated")
+        tags = Set.new
+        if (detected_tag = detect_tag(deprecation))
+          tags << detected_tag
+        end
+        tags << deprecation[:realm] if deprecation[:realm] && deprecation[:realm] != "rails"
+        tags.merge(deprecation.dig(:notes, :tags) || [])
 
-          tags[:test] = 'bg-success' if deprecation[:message].include?("trigger_kwargs_error_warning") ||
-                                        deprecation[:message].include?("trigger_rails_deprecation")
-            
-          tags[deprecation[:realm]] = 'bg-secondary' if deprecation[:realm] && deprecation[:realm] != 'rails'
+        tags.to_h do |tag|
+          next [tag, "bg-success"] if tag == :test
 
-          deprecation.dig(:notes, :tags)&.each { |tag| tags[tag] = 'bg-secondary' }
+          [tag, "bg-secondary"]
         end
       end
     end