deprecation_collector 0.3.0 → 0.5.0

data/gemfiles/rails_7.gemfile.lock

@@ -1,73 +1,73 @@
 PATH
   remote: ..
   specs:
-    deprecation_collector (0.2.0)
+    deprecation_collector (0.5.0)
       redis (>= 3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.4.1)
-      actionpack (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    actioncable (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.4.1)
-      actionpack (= 7.0.4.1)
-      activejob (= 7.0.4.1)
-      activerecord (= 7.0.4.1)
-      activestorage (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    actionmailbox (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activestorage (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.4.1)
-      actionpack (= 7.0.4.1)
-      actionview (= 7.0.4.1)
-      activejob (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    actionmailer (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      actionview (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.4.1)
-      actionview (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    actionpack (7.0.4.3)
+      actionview (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       rack (~> 2.0, >= 2.2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.4.1)
-      actionpack (= 7.0.4.1)
-      activerecord (= 7.0.4.1)
-      activestorage (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    actiontext (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activestorage (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.4.1)
-      activesupport (= 7.0.4.1)
+    actionview (7.0.4.3)
+      activesupport (= 7.0.4.3)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (7.0.4.1)
-      activesupport (= 7.0.4.1)
+    activejob (7.0.4.3)
+      activesupport (= 7.0.4.3)
       globalid (>= 0.3.6)
-    activemodel (7.0.4.1)
-      activesupport (= 7.0.4.1)
-    activerecord (7.0.4.1)
-      activemodel (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
-    activestorage (7.0.4.1)
-      actionpack (= 7.0.4.1)
-      activejob (= 7.0.4.1)
-      activerecord (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    activemodel (7.0.4.3)
+      activesupport (= 7.0.4.3)
+    activerecord (7.0.4.3)
+      activemodel (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+    activestorage (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.4.1)
+    activesupport (7.0.4.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -77,21 +77,21 @@ GEM
       rake
       thor (>= 0.14.0)
     builder (3.2.4)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.2)
     crass (1.0.6)
     date (3.3.3)
     diff-lcs (1.5.0)
     erubi (1.12.0)
-    fakeredis (0.8.0)
-      redis (~> 4.1)
-    globalid (1.0.0)
+    fakeredis (0.9.1)
+      redis (~> 4.8)
+    globalid (1.1.0)
       activesupport (>= 5.0)
-    i18n (1.12.0)
+    i18n (1.13.0)
       concurrent-ruby (~> 1.0)
-    loofah (2.19.1)
+    loofah (2.20.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    mail (2.8.0.1)
+    mail (2.8.1)
       mini_mime (>= 0.1.1)
       net-imap
       net-pop
@@ -99,7 +99,7 @@ GEM
     marcel (1.0.2)
     method_source (1.0.0)
     mini_mime (1.1.2)
-    minitest (5.17.0)
+    minitest (5.18.0)
     net-imap (0.3.4)
       date
       net-protocol
@@ -109,65 +109,71 @@ GEM
       timeout
     net-smtp (0.3.3)
       net-protocol
-    nio4r (2.5.8)
-    nokogiri (1.14.0-x86_64-darwin)
+    nio4r (2.5.9)
+    nokogiri (1.14.3-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.14.0-x86_64-linux)
+    nokogiri (1.14.3-x86_64-linux)
       racc (~> 1.4)
     racc (1.6.2)
-    rack (2.2.6.1)
-    rack-test (2.0.2)
+    rack (2.2.7)
+    rack-test (2.1.0)
       rack (>= 1.3)
-    rails (7.0.4.1)
-      actioncable (= 7.0.4.1)
-      actionmailbox (= 7.0.4.1)
-      actionmailer (= 7.0.4.1)
-      actionpack (= 7.0.4.1)
-      actiontext (= 7.0.4.1)
-      actionview (= 7.0.4.1)
-      activejob (= 7.0.4.1)
-      activemodel (= 7.0.4.1)
-      activerecord (= 7.0.4.1)
-      activestorage (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    rails (7.0.4.3)
+      actioncable (= 7.0.4.3)
+      actionmailbox (= 7.0.4.3)
+      actionmailer (= 7.0.4.3)
+      actionpack (= 7.0.4.3)
+      actiontext (= 7.0.4.3)
+      actionview (= 7.0.4.3)
+      activejob (= 7.0.4.3)
+      activemodel (= 7.0.4.3)
+      activerecord (= 7.0.4.3)
+      activestorage (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       bundler (>= 1.15.0)
-      railties (= 7.0.4.1)
+      railties (= 7.0.4.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.4)
+    rails-html-sanitizer (1.5.0)
       loofah (~> 2.19, >= 2.19.1)
-    railties (7.0.4.1)
-      actionpack (= 7.0.4.1)
-      activesupport (= 7.0.4.1)
+    railties (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rake (13.0.6)
-    redis (4.8.0)
+    redis (4.8.1)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.0)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.3)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
+    slim (5.1.0)
+      temple (~> 0.10.0)
+      tilt (>= 2.0.6, < 2.2)
+    temple (0.10.0)
     thor (1.2.1)
+    tilt (2.1.0)
     timecop (0.9.6)
-    timeout (0.3.1)
-    tzinfo (2.0.5)
+    timeout (0.3.2)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    webrick (1.8.1)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.6.6)
+    zeitwerk (2.6.7)
 
 PLATFORMS
   x86_64-darwin-21
@@ -177,11 +183,15 @@ DEPENDENCIES
   appraisal
   deprecation_collector!
   fakeredis
+  rack
+  rack-test
   rails (~> 7.0)
   rake (~> 13.0)
   redis (~> 4.8)
   rspec (~> 3.0)
+  slim
   timecop
+  webrick
 
 BUNDLED WITH
    2.3.10

data/gemfiles/rails_none.gemfile

@@ -7,5 +7,8 @@ gem "rspec", "~> 3.0"
 gem "timecop"
 gem "fakeredis"
 gem "redis", "~>4.8"
+gem "rack"
+gem "webrick"
+gem "slim"
 
 gemspec path: "../"

data/gemfiles/rails_none.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    deprecation_collector (0.2.0)
+    deprecation_collector (0.5.0)
       redis (>= 3.0)
 
 GEM
@@ -12,25 +12,34 @@ GEM
       rake
       thor (>= 0.14.0)
     diff-lcs (1.5.0)
-    fakeredis (0.8.0)
-      redis (~> 4.1)
+    fakeredis (0.9.1)
+      redis (~> 4.8)
+    rack (3.0.7)
+    rack-test (2.1.0)
+      rack (>= 1.3)
     rake (13.0.6)
-    redis (4.8.0)
+    redis (4.8.1)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.0)
+    rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.2)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.3)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
+    slim (5.1.0)
+      temple (~> 0.10.0)
+      tilt (>= 2.0.6, < 2.2)
+    temple (0.10.0)
     thor (1.2.1)
+    tilt (2.1.0)
     timecop (0.9.6)
+    webrick (1.8.1)
 
 PLATFORMS
   x86_64-darwin-21
@@ -40,10 +49,14 @@ DEPENDENCIES
   appraisal
   deprecation_collector!
   fakeredis
+  rack
+  rack-test
   rake (~> 13.0)
   redis (~> 4.8)
   rspec (~> 3.0)
+  slim
   timecop
+  webrick
 
 BUNDLED WITH
    2.3.10

data/lib/deprecation_collector/deprecation.rb

@@ -4,7 +4,7 @@ class DeprecationCollector
   # :nodoc:
   class Deprecation
     attr_reader :message, :realm, :gem_traceline, :app_traceline, :occurences, :first_timestamp, :full_backtrace
-    attr_accessor :context, :custom_fingerprint
+    attr_accessor :context, :custom_fingerprint, :app_name
 
     CLEANUP_REGEXES = {
       # rails views generated methods names are unique per-worker
@@ -61,6 +61,7 @@ class DeprecationCollector
 
     def as_json(_options = {})
       {
+        app: app_name,
         message: message,
         realm: realm,
         app_traceline: app_traceline,

data/lib/deprecation_collector/storage.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require "redis"
+
+class DeprecationCollector
+  module Storage
+    # :nodoc:
+    class Base
+      # rubocop:disable Style/SingleLineMethods
+      def enabled?; true; end
+      def enable; end
+      def disable; end
+
+      def unsent_deprecations; []; end
+      def fetch_known_digests; end
+
+      def delete(digests); end
+      def clear(enable: false); end
+
+      def store(_deprecation); raise("Not implemented"); end
+      # rubocop:enable Style/SingleLineMethods
+    end
+
+    # dummy strategy that outputs every deprecation into stderr
+    class StdErr < Base
+      def store(deprecation)
+        DeprecationCollector.instance.send(:log_deprecation, deprecation)
+      end
+    end
+
+    # storing in redis with deduplication by fingerprint
+    class Redis < Base
+      attr_accessor :write_interval, :write_interval_jitter, :redis, :count
+
+      def initialize(redis, mutex: nil, count: false, write_interval: 900, write_interval_jitter: 60,
+                            key_prefix: nil)
+        super()
+        @key_prefix = key_prefix || "deprecations"
+        @redis = redis
+        @last_write_time = current_time
+        @count = count
+        @write_interval = write_interval
+        @write_interval_jitter = write_interval_jitter
+        # on cruby hash itself is threadsafe, but we need to prevent races
+        @deprecations_mutex = mutex || Mutex.new
+        @deprecations = {}
+        @known_digests = Set.new
+      end
+
+      def unsent_deprecations
+        @deprecations
+      end
+
+      def enabled?
+        @redis.get(enabled_flag_key) != "false"
+      end
+
+      def enable
+        @redis.set(enabled_flag_key, "true")
+      end
+
+      def disable
+        @redis.set(enabled_flag_key, "false")
+      end
+
+      def delete(remove_digests)
+        return 0 unless remove_digests.any?
+
+        @redis.pipelined do |pipe|
+          pipe.hdel(data_hash_key, *remove_digests)
+          pipe.hdel(notes_hash_key, *remove_digests)
+          pipe.hdel(counter_hash_key, *remove_digests) if @count
+        end.first
+      end
+
+      def clear(enable: false)
+        @redis.del(data_hash_key, counter_hash_key, notes_hash_key)
+        @redis.del(enabled_flag_key) if enable
+        @known_digests.clear
+        @deprecations.clear
+      end
+
+      def fetch_known_digests
+        # FIXME: use `.merge!`?
+        @known_digests.merge(@redis.hkeys(data_hash_key))
+      end
+
+      def store(deprecation)
+        fresh = !@deprecations.key?(deprecation.digest)
+        @deprecations_mutex.synchronize do
+          (@deprecations[deprecation.digest] ||= deprecation).touch
+        end
+
+        flush if current_time - @last_write_time > (@write_interval + rand(@write_interval_jitter))
+        fresh
+      end
+
+      def flush(force: false)
+        return unless force || (current_time > @last_write_time + @write_interval)
+
+        deprecations_to_flush = nil
+        @deprecations_mutex.synchronize do
+          deprecations_to_flush = @deprecations
+          @deprecations = {}
+          @last_write_time = current_time
+          # checking in this section to prevent multiple parallel check requests
+          return DeprecationCollector.instance.instance_variable_set(:@enabled, false) unless enabled?
+        end
+
+        write_count_to_redis(deprecations_to_flush) if @count
+
+        # make as few writes as possible, other workers may already have reported our warning
+        fetch_known_digests
+        deprecations_to_flush.reject! { |digest, _val| @known_digests.include?(digest) }
+        return unless deprecations_to_flush.any?
+
+        @known_digests.merge(deprecations_to_flush.keys)
+        @redis.mapped_hmset(data_hash_key, deprecations_to_flush.transform_values(&:to_json))
+      end
+
+      def read_each
+        cursor = 0
+        loop do
+          cursor, data_pairs = @redis.hscan(data_hash_key, cursor)
+
+          if data_pairs.any?
+            data_pairs.zip(
+              @redis.hmget(counter_hash_key, data_pairs.map(&:first)),
+              @redis.hmget(notes_hash_key, data_pairs.map(&:first))
+            ).each do |(digest, data), count, notes|
+              yield(digest, data, count, notes)
+            end
+          end
+          break if cursor == "0"
+        end
+      end
+
+      def read_one(digest)
+        [
+          digest,
+          *@redis.pipelined do |pipe|
+            pipe.hget(data_hash_key, digest)
+            pipe.hget(counter_hash_key, digest)
+            pipe.hget(notes_hash_key, digest)
+          end
+        ]
+      end
+
+      def import(dump_hash)
+        @redis.mapped_hmset(data_hash_key, dump_hash.transform_values(&:to_json))
+      end
+
+      def cleanup(&_block)
+        cursor = 0
+        removed = total = 0
+        loop do
+          cursor, data_pairs = @redis.hscan(data_hash_key, cursor) # NB: some pages may be empty
+          total += data_pairs.size
+          removed += delete(
+            data_pairs.to_h.select { |_digest, data| yield(JSON.parse(data, symbolize_names: true)) }.keys
+          )
+          break if cursor == "0"
+        end
+        "#{removed} removed, #{total - removed} left"
+      end
+
+      def key_prefix=(val)
+        @enabled_flag_key = @data_hash_key = @counter_hash_key = @notes_hash_key = nil
+        @key_prefix = val
+      end
+
+      protected
+
+      def enabled_flag_key
+        @enabled_flag_key ||= "#{@key_prefix}:enabled" # usually deprecations:enabled
+      end
+
+      def data_hash_key
+        @data_hash_key ||= "#{@key_prefix}:data" # usually deprecations:data
+      end
+
+      def counter_hash_key
+        @counter_hash_key ||= "#{@key_prefix}:counter" # usually deprecations:counter
+      end
+
+      def notes_hash_key
+        @notes_hash_key ||= "#{@key_prefix}:notes" # usually deprecations:notes
+      end
+
+      def current_time
+        return Time.zone.now if Time.respond_to?(:zone) && Time.zone
+
+        Time.now
+      end
+
+      def write_count_to_redis(deprecations_to_flush)
+        @redis.pipelined do |pipe|
+          deprecations_to_flush.each_pair do |digest, deprecation|
+            pipe.hincrby(counter_hash_key, digest, deprecation.occurences)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deprecation_collector/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class DeprecationCollector
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/deprecation_collector/web/application.rb

@@ -1,29 +1,30 @@
 # frozen_string_literal: true
 
-require_relative 'router'
-require_relative 'helpers'
-require 'pp'
+require_relative "router"
+require_relative "helpers"
 
 class DeprecationCollector
   class Web
+    # :nodoc:
     class Application
       extend Web::Router
       helpers Helpers
 
-      def initialize
-        unless defined?(Temple::Utils) || ENV['DEPRECATION_COLLECTOR_RELOAD_WEB_TEMPLATES']
-          # used for escaping in compiled slim templates
-          require_relative 'utils'
-        end
+      attr_reader :web
+
+      def initialize(web)
+        @web = web
+        # used for escaping in compiled slim templates
+        require_relative "utils" unless defined?(Temple::Utils) || ENV["DEPRECATION_COLLECTOR_RELOAD_WEB_TEMPLATES"]
       end
 
       def call(env)
-        self.class.call(env)
+        self.class.call(env, self)
       end
 
       root do # index
-        @deprecations = collector_instance.read_each.to_a
-        @deprecations = @deprecations.sort_by { |dep| dep[:message] } unless params[:sort] == '0'
+        @deprecations = collector_instance.read_each.to_a.compact
+        @deprecations = @deprecations.sort_by { |dep| dep[:message] } unless params[:sort] == "0"
 
         if params[:reject]
           @deprecations = @deprecations.reject { |dep| dep[:message].match?(Regexp.union(Array(params[:reject]))) }
@@ -36,38 +37,59 @@ class DeprecationCollector
         render slim: "index.html"
       end
 
-      get '/dump.json' do
-        render json: collector_instance.read_each.to_a
+      get "/dump.json" do
+        render json: collector_instance.dump
+      end
+
+      get "/import" do
+        return "Import not enabled" unless import_enabled?
+
+        render slim: "import.html"
+      end
+
+      post "/import" do
+        unless env["CONTENT_TYPE"]&.start_with?("multipart/form-data") && params.dig(:file, :tempfile)
+          halt 422, "need multipart json file"
+        end
+        collector_instance.import_dump(File.read(params[:file][:tempfile]))
+        redirect_to deprecations_path
+      end
+
+      get "/:id.json" do # show
+        @deprecation = collector_instance.read_one(params[:id])
+        halt 404 unless @deprecation
+        render json: JSON.pretty_generate(@deprecation)
       end
 
-      get '/:id' do # show
+      get "/:id" do # show
         @deprecation = collector_instance.read_one(params[:id])
+        halt 404 unless @deprecation
         render slim: "show.html"
       end
 
-      delete '/all' do
+      delete "/all" do
         collector_instance.flush_redis
         redirect_to deprecations_path
       end
 
-      post '/enable' do
+      post "/enable" do
         collector_instance.enable
         redirect_to deprecations_path
       end
 
-      delete '/disable' do
+      delete "/disable" do
         collector_instance.disable
         redirect_to deprecations_path
       end
 
       # NB: order for wildcards is important
-      delete '/:id' do # destroy
+      delete "/:id" do # destroy
         collector_instance.delete_deprecations([params[:id]])
         redirect_to deprecations_path
       end
 
-      post '/trigger' do # trigger
-        trigger_kwargs_error_warning({ foo: nil }) if RUBY_VERSION.start_with?('2.7')
+      post "/trigger" do # trigger
+        trigger_kwargs_error_warning({ foo: nil }) if RUBY_VERSION.start_with?("2.7")
         trigger_rails_deprecation
         collector_instance.collect(
           "TestFoo#assign_attributes called (test attr_spy) trigger_rails_deprecation", caller_locations, :attr_spy