logster 2.4.2 → 2.5.0

data/lib/logster/logger.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'logger'
 
 module Logster

data/lib/logster/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'digest/sha1'
 require 'securerandom'
 
@@ -112,7 +114,7 @@ module Logster
 
     # in its own method so it can be overridden
     def grouping_hash
-      return { message: self.message, severity: self.severity, backtrace: self.backtrace }
+      { message: self.message, severity: self.severity, backtrace: self.backtrace }
     end
 
     # todo - memoize?

data/lib/logster/middleware/reporter.rb

@@ -25,14 +25,14 @@ module Logster
         if path == @error_path
 
           if !Logster.config.enable_js_error_reporting
-            return [403, {}, "Access Denied"]
+            return [403, {}, ["Access Denied"]]
           end
 
           Logster.config.current_context.call(env) do
             if Logster.config.rate_limit_error_reporting
               req = Rack::Request.new(env)
               if Logster.store.rate_limited?(req.ip, perform: true)
-                return [429, {}, "Rate Limited"]
+                return [429, {}, ["Rate Limited"]]
               end
             end
             report_js_error(env)

data/lib/logster/middleware/viewer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 
 module Logster
@@ -122,7 +124,11 @@ module Logster
                 count = ignore_count[pattern] || 0
                 suppression << { value: pattern, count: count }
               end
-              [200, { "Content-Type" => "application/json; charset=utf-8" }, [JSON.generate(suppression: suppression)]]
+
+              grouping = Logster::GroupingPattern.find_all(raw: true).map do |pattern|
+                { value: pattern }
+              end
+              [200, { "Content-Type" => "application/json; charset=utf-8" }, [JSON.generate(suppression: suppression, grouping: grouping)]]
             else
               [200, { "Content-Type" => "text/html; charset=utf-8" }, [body(preload_json)]]
             end
@@ -199,6 +205,9 @@ module Logster
           opts[:search] = search
         end
         search = opts[:search]
+        if params["known_groups"]
+          opts[:known_groups] = params["known_groups"]
+        end
         opts[:with_env] = (String === search && search.size > 0) || Regexp === search
 
         payload = {
@@ -253,6 +262,8 @@ module Logster
         case set_name
         when "suppression"
           Logster::SuppressionPattern
+        when "grouping"
+          Logster::GroupingPattern
         else
           nil
         end

data/lib/logster/pattern.rb

@@ -1,7 +1,20 @@
+# frozen_string_literal: true
+
 module Logster
   class Pattern
+    @child_classes = []
+
     class PatternError < StandardError; end
 
+    def self.inherited(subclass)
+      @child_classes << subclass
+      super
+    end
+
+    def self.child_classes
+      @child_classes
+    end
+
     def self.set_name
       raise "Please override the `set_name` method and specify and a name for this set"
     end

data/lib/logster/redis_store.rb

@@ -39,7 +39,12 @@ module Logster
     end
 
     def delete(msg)
+      groups = find_pattern_groups() { |pat| msg.message =~ pat }
       @redis.multi do
+        groups.each do |group|
+          group.remove_message(msg)
+          save_pattern_group(group) if group.changed?
+        end
         @redis.hdel(hash_key, msg.key)
         @redis.hdel(env_key, msg.key)
         @redis.hdel(grouping_key, msg.grouping_key)
@@ -48,7 +53,12 @@ module Logster
     end
 
     def bulk_delete(message_keys, grouping_keys)
+      groups = find_pattern_groups(load_messages: true)
       @redis.multi do
+        groups.each do |group|
+          group.messages = group.messages.reject { |m| message_keys.include?(m.key) }
+          save_pattern_group(group) if group.changed?
+        end
         @redis.hdel(hash_key, message_keys)
         @redis.hdel(env_key, message_keys)
         @redis.hdel(grouping_key, grouping_keys)
@@ -100,19 +110,21 @@ module Logster
       after = opts[:after]
       search = opts[:search]
       with_env = opts.key?(:with_env) ? opts[:with_env] : true
+      known_groups = opts[:known_groups]&.dup || []
 
       start, finish = find_location(before, after, limit)
 
       return [] unless start && finish
 
       results = []
+      pattern_groups = find_pattern_groups(load_messages: true)
 
       direction = after ? 1 : -1
 
       begin
         keys = @redis.lrange(list_key, start, finish) || []
-        break unless keys && (keys.count > 0)
-        rows = bulk_get(keys, with_env: with_env)
+        break if !keys || keys.count <= 0
+        rows = bulk_get(keys, with_env: with_env).reverse
 
         temp = []
 
@@ -121,9 +133,18 @@ module Logster
           row = nil if severity && !severity.include?(row.severity)
 
           row = filter_search(row, search)
-          temp << row if row
+          if row
+            group = pattern_groups.find { |g| g.messages_keys.include?(row.key) }
+            if group && !known_groups.include?(group.key)
+              known_groups << group.key
+              temp << serialize_group(group, row.key)
+            elsif !group
+              temp << row
+            end
+          end
         end
 
+        temp.reverse!
         if direction == -1
           results = temp + results
         else
@@ -147,6 +168,8 @@ module Logster
       if keys.empty?
         @redis.del(hash_key)
         @redis.del(env_key)
+        @redis.del(pattern_groups_key)
+        @redis.del(grouping_key)
       else
         protected = @redis.mapped_hmget(hash_key, *keys)
         protected_env = @redis.mapped_hmget(env_key, *keys)
@@ -169,6 +192,10 @@ module Logster
             @redis.rpush(list_key, message_key)
           end
         end
+        find_pattern_groups(load_messages: true).each do |group|
+          group.messages = group.messages.select { |m| sorted.include?(m.key) }
+          save_pattern_group(group) if group.changed?
+        end
       end
     end
 
@@ -182,7 +209,8 @@ module Logster
       @redis.del(grouping_key)
       @redis.del(solved_key)
       @redis.del(ignored_logs_count_key)
-      Logster::PATTERNS.each do |klass|
+      @redis.del(pattern_groups_key)
+      Logster::Pattern.child_classes.each do |klass|
         @redis.del(klass.set_name)
       end
       @redis.keys.each do |key|
@@ -202,13 +230,15 @@ module Logster
       message
     end
 
-    def get_all_messages
-      bulk_get(@redis.lrange(list_key, 0, -1))
+    def get_all_messages(with_env: true)
+      bulk_get(@redis.lrange(list_key, 0, -1), with_env: with_env)
     end
 
     def bulk_get(message_keys, with_env: true)
+      return [] if !message_keys || message_keys.size == 0
       envs = @redis.mapped_hmget(env_key, *message_keys) if with_env
-      @redis.hmget(hash_key, message_keys).map! do |json|
+      messages = @redis.hmget(hash_key, message_keys).map! do |json|
+        next if !json || json.size == 0
         message = Message.from_json(json)
         if with_env
           env = envs[message.key]
@@ -219,6 +249,8 @@ module Logster
         end
         message
       end
+      messages.compact!
+      messages
     end
 
     def get_env(message_key)
@@ -301,6 +333,45 @@ module Logster
       limited
     end
 
+    def find_pattern_groups(load_messages: false)
+      patterns = @patterns_cache.fetch(Logster::GroupingPattern::CACHE_KEY) do
+        Logster::GroupingPattern.find_all(store: self)
+      end
+      patterns = patterns.select do |pattern|
+        if block_given?
+          yield(pattern)
+        else
+          true
+        end
+      end
+      return [] if patterns.size == 0
+      mapped = patterns.map(&:inspect)
+      jsons = @redis.hmget(pattern_groups_key, mapped)
+      jsons.map! do |json|
+        if json && json.size > 0
+          group = Logster::Group.from_json(json)
+          if load_messages
+            group.messages = bulk_get(group.messages_keys, with_env: false)
+          end
+          group
+        end
+      end
+      jsons.compact!
+      jsons
+    end
+
+    def save_pattern_group(group)
+      if group.count == 0
+        @redis.hdel(pattern_groups_key, group.key)
+      else
+        @redis.hset(pattern_groups_key, group.key, group.to_json)
+      end
+    end
+
+    def remove_pattern_group(pattern)
+      @redis.hdel(pattern_groups_key, pattern.inspect)
+    end
+
     protected
 
     def clear_solved(count = nil)
@@ -325,9 +396,7 @@ module Logster
         while removed_key = @redis.lpop(list_key)
           unless @redis.sismember(protected_key, removed_key)
             rmsg = get removed_key
-            @redis.hdel(hash_key, rmsg.key)
-            @redis.hdel(env_key, rmsg.key)
-            @redis.hdel(grouping_key, rmsg.grouping_key)
+            delete(rmsg)
             break
           else
             removed_keys << removed_key
@@ -515,8 +584,28 @@ module Logster
       "__LOGSTER__IP_RATE_LIMIT_#{ip_address}"
     end
 
+    def pattern_groups_key
+      @pattern_groups_key ||= "__LOGSTER__PATTERN_GROUPS_KEY__MAP"
+    end
+
     private
 
+    def serialize_group(group, row_id)
+      # row_id should be the key of the most recent *message* that is
+      # included in the group.
+      # It's used by the client in the before (not after) query param
+      # when you hit load more and the first row is a group.
+      # The server uses this info (row_id) to know where it needs to
+      # start scanning messages when looking up older messages.
+      Logster::Group::GroupWeb.new(
+        group.key,
+        group.count,
+        group.timestamp,
+        group.messages,
+        row_id
+      )
+    end
+
     def apply_max_size_limit(message)
       size = message.to_json(exclude_env: true).bytesize
       env_size = message.env_json.bytesize

data/lib/logster/scheduler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Logster
   module Deferer
     attr_reader :queue, :thread

data/lib/logster/suppression_pattern.rb

@@ -1,19 +1,22 @@
+# frozen_string_literal: true
+
 module Logster
   class SuppressionPattern < Pattern
+    CACHE_KEY = :suppression
     def self.set_name
       "__LOGSTER__suppression_patterns_set".freeze
     end
 
     def save(args = {})
       super
-      @store.clear_suppression_patterns_cache
+      @store.clear_patterns_cache(CACHE_KEY)
       retro_delete_messages if args[:retroactive]
     end
 
     def destroy(clear_cache: true) # arg used in tests
       super()
       @store.remove_ignore_count(self.to_s)
-      @store.clear_suppression_patterns_cache if clear_cache
+      @store.clear_patterns_cache(CACHE_KEY) if clear_cache
     end
 
     private

data/lib/logster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Logster
-  VERSION = "2.4.2"
+  VERSION = "2.5.0"
 end

data/lib/logster/web.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'logster/middleware/viewer'
 
 class Logster::Web

data/logster.gemspec

@@ -1,4 +1,6 @@
 # coding: utf-8
+# frozen_string_literal: true
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'logster/version'
@@ -31,5 +33,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "guard-minitest"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "byebug"
-  spec.add_development_dependency "rubocop", "~> 0.61.1"
+  spec.add_development_dependency "rubocop", "~> 0.69.0"
+  spec.add_development_dependency "rubocop-discourse"
 end

data/test/examples/test_sidekiq_reporter_example.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../test_helper'
 require 'logster/logger'
 require 'logster/redis_store'

data/test/fake_data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gem 'redis'

data/test/fake_data/generate.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'redis'
 require 'logster'
 

data/test/logster/middleware/test_viewer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../test_helper'
 require 'rack'
 require 'logster/redis_store'
@@ -126,7 +128,7 @@ class TestViewer < Minitest::Test
       params: { pattern: "disallowedpattern" }
     )
     assert_equal(404, response.status)
-    Logster::PATTERNS.each do |klass|
+    Logster::Pattern.child_classes.each do |klass|
       assert_equal(0, klass.find_all.size)
     end
   ensure

data/test/logster/test_base_store.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../test_helper'
 require 'logster/base_store'
 require 'logster/ignore_pattern'

data/test/logster/test_cache.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../test_helper'
 require 'logster/cache'
 
@@ -7,32 +9,37 @@ class TestCache < Minitest::Test
   end
 
   def test_cache_works
-    value = "I should be retured"
-    prc = Proc.new do
-      @cache.fetch do
+    prc = Proc.new do |key, value|
+      @cache.fetch(key) do
         value
       end
     end
-    assert_equal(value, prc.call)
+    value = "I should be retured"
+    assert_equal(value, prc.call(:key1, value))
     cached_value = value
     value = "I shouldn't be returned"
-    assert_equal(cached_value, prc.call)
+    assert_equal(cached_value, prc.call(:key1, value))
+    value2 = "value for key2"
+    assert_equal(value2, prc.call(:key2, value2))
 
-    value = "Now I should be returned again"
+    value = value2 = "Now I should be returned"
     Process.stub :clock_gettime,  Process.clock_gettime(Process::CLOCK_MONOTONIC) + 6 do
-      assert_equal(value, prc.call)
+      assert_equal(value, prc.call(:key1, value))
+      assert_equal(value2, prc.call(:key2, value2))
     end
   end
 
   def test_cache_can_be_cleared
     value = "cached"
-    prc = Proc.new do
-      @cache.fetch { value }
+    prc = Proc.new do |key, val|
+      @cache.fetch(key) { val }
     end
-    assert_equal(value, prc.call)
+    assert_equal(value, prc.call(:key1, value))
+    assert_equal("v2", prc.call(:key2, "v2"))
 
     value = "new value"
-    @cache.clear
-    assert_equal(value, prc.call)
+    @cache.clear(:key1)
+    assert_equal(value, prc.call(:key1, value))
+    assert_equal("v2", prc.call(:key2, "v2.2"))
   end
 end