logster 2.5.1 → 2.6.0

data/lib/logster/middleware/viewer.rb

@@ -38,7 +38,7 @@ module Logster
 
           elsif resource =~ /\/message\/([0-9a-f]+)$/
             if env[REQUEST_METHOD] != "DELETE"
-              return method_not_allowed("DELETE is needed for /clear")
+              return method_not_allowed("DELETE")
             end
 
             key = $1
@@ -87,7 +87,7 @@ module Logster
 
           elsif resource =~ /\/clear$/
             if env[REQUEST_METHOD] != "POST"
-              return method_not_allowed("POST is needed for /clear")
+              return method_not_allowed("POST")
             end
             Logster.store.clear
             return [200, {}, ["Messages cleared"]]
@@ -139,12 +139,12 @@ module Logster
 
             set_name = $1
             req = Rack::Request.new(env)
-            return method_not_allowed if req.request_method == "GET"
+            return method_not_allowed(%w[POST PUT DELETE]) if req.request_method == "GET"
 
             update_patterns(set_name, req)
           elsif resource == "/reset-count.json"
             req = Rack::Request.new(env)
-            return method_not_allowed("PUT is needed for this endpoint") if req.request_method != "PUT"
+            return method_not_allowed("PUT") if req.request_method != "PUT"
             pattern = nil
             if [true, "true"].include?(req.params["hard"])
               pattern = Logster.store.ignore.find do |patt|
@@ -170,6 +170,16 @@ module Logster
             else
               not_found
             end
+          elsif resource == '/solve-group'
+            return not_allowed unless Logster.config.enable_custom_patterns_via_ui
+            req = Rack::Request.new(env)
+            return method_not_allowed("POST") if req.request_method != "POST"
+            group = Logster.store.find_pattern_groups do |patt|
+              patt.inspect == req.params["regex"]
+            end.first
+            return not_found("No such pattern group exists") if !group
+            group.messages_keys.each { |k| Logster.store.solve(k) }
+            return [200, {}, []]
           else
             not_found
           end
@@ -243,7 +253,7 @@ module Logster
         when "DELETE"
           record.destroy
         else
-          return method_not_allowed("Allowed methods: POST, PUT or DELETE")
+          return method_not_allowed(%w[POST PUT DELETE])
         end
 
         [200, { "Content-Type" => "application/json" }, [JSON.generate(pattern: record.to_s)]]
@@ -277,8 +287,11 @@ module Logster
         [403, {}, [message]]
       end
 
-      def method_not_allowed(message = "Method not allowed")
-        [405, {}, [message]]
+      def method_not_allowed(allowed_methods)
+        if Array === allowed_methods
+          allowed_methods = allowed_methods.join(", ")
+        end
+        [405, { "Allow" => allowed_methods }, []]
       end
 
       def parse_regex(string)
@@ -317,13 +330,36 @@ module Logster
         Rack::Utils.escape_html(JSON.fast_generate(payload))
       end
 
+      def preload_backtrace_data
+        gems_data = []
+        Gem::Specification.find_all do |gem|
+          url = gem.metadata["source_code_uri"] || gem.homepage
+          if url && url.match(/^https?:\/\/github.com\//)
+            gems_data << { name: gem.name, url: url }
+          end
+        end
+        {
+          gems_data: gems_data,
+          directories: Logster.config.project_directories
+        }
+      end
+
       def body(preload)
         root_url = @logs_path
         root_url += "/" if root_url[-1] != "/"
         preload.merge!(
           env_expandable_keys: Logster.config.env_expandable_keys,
-          patterns_enabled: Logster.config.enable_custom_patterns_via_ui
+          patterns_enabled: Logster.config.enable_custom_patterns_via_ui,
+          application_version: Logster.config.application_version
         )
+        backtrace_links_enabled = Logster.config.enable_backtrace_links
+        gems_dir = Logster.config.gems_dir
+        gems_dir += "/" if gems_dir[-1] != "/"
+        preload.merge!(gems_dir: gems_dir, backtrace_links_enabled: backtrace_links_enabled)
+
+        if backtrace_links_enabled
+          preload.merge!(preload_backtrace_data)
+        end
         <<~HTML
           <!doctype html>
           <html>

data/lib/logster/redis_store.rb

@@ -6,6 +6,7 @@ require 'logster/redis_rate_limiter'
 
 module Logster
   class RedisStore < BaseStore
+    ENV_PREFIX = "logster-env-"
 
     attr_accessor :redis, :max_backlog, :redis_raw_connection
     attr_writer :redis_prefix
@@ -21,15 +22,14 @@ module Logster
     def save(message)
       if keys = message.solved_keys
         keys.each do |solved|
-          return true if @redis.hget(solved_key, solved)
+          return false if @redis.hget(solved_key, solved)
         end
       end
-      apply_max_size_limit(message)
 
       @redis.multi do
         @redis.hset(grouping_key, message.grouping_key, message.key)
         @redis.rpush(list_key, message.key)
-        update_message(message)
+        update_message(message, save_env: true)
       end
 
       trim
@@ -46,7 +46,7 @@ module Logster
           save_pattern_group(group) if group.changed?
         end
         @redis.hdel(hash_key, msg.key)
-        @redis.hdel(env_key, msg.key)
+        delete_env(msg.key)
         @redis.hdel(grouping_key, msg.grouping_key)
         @redis.lrem(list_key, -1, msg.key)
       end
@@ -60,26 +60,26 @@ module Logster
           save_pattern_group(group) if group.changed?
         end
         @redis.hdel(hash_key, message_keys)
-        @redis.hdel(env_key, message_keys)
         @redis.hdel(grouping_key, grouping_keys)
         message_keys.each do |k|
           @redis.lrem(list_key, -1, k)
+          delete_env(k)
         end
       end
     end
 
-    def replace_and_bump(message, save_env: true)
+    def replace_and_bump(message)
       # TODO make it atomic
       exists = @redis.hexists(hash_key, message.key)
       return false unless exists
 
       @redis.multi do
         @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
-        @redis.hset(env_key, message.key, message.env_json) if save_env
+        push_env(message.key, message.env_buffer) if message.has_env_buffer?
         @redis.lrem(list_key, -1, message.key)
         @redis.rpush(list_key, message.key)
       end
-
+      message.env_buffer = [] if message.has_env_buffer?
       check_rate_limits(message.severity)
 
       true
@@ -164,22 +164,21 @@ module Logster
     def clear
       RedisRateLimiter.clear_all(@redis)
       @redis.del(solved_key)
+      all_keys = @redis.lrange(list_key, 0, -1)
       @redis.del(list_key)
-      keys = @redis.smembers(protected_key) || []
-      if keys.empty?
+      protected_keys = @redis.smembers(protected_key) || []
+      if protected_keys.empty?
         @redis.del(hash_key)
-        @redis.del(env_key)
+        all_keys.each { |k| delete_env(k) }
         @redis.del(pattern_groups_key)
         @redis.del(grouping_key)
       else
-        protected = @redis.mapped_hmget(hash_key, *keys)
-        protected_env = @redis.mapped_hmget(env_key, *keys)
+        protected_messages = @redis.mapped_hmget(hash_key, *protected_keys)
         @redis.del(hash_key)
-        @redis.del(env_key)
-        @redis.mapped_hmset(hash_key, protected)
-        @redis.mapped_hmset(env_key, protected_env)
+        @redis.mapped_hmset(hash_key, protected_messages)
+        (all_keys - protected_keys).each { |k| delete_env(k) }
 
-        sorted = protected
+        sorted = protected_messages
           .values
           .map { |string|
             Message.from_json(string) rescue nil
@@ -203,10 +202,10 @@ module Logster
     # Delete everything, included protected messages
     # (use in tests)
     def clear_all
+      @redis.lrange(list_key, 0, -1).each { |k| delete_env(k) }
       @redis.del(list_key)
       @redis.del(protected_key)
       @redis.del(hash_key)
-      @redis.del(env_key)
       @redis.del(grouping_key)
       @redis.del(solved_key)
       @redis.del(ignored_logs_count_key)
@@ -235,17 +234,35 @@ module Logster
       bulk_get(@redis.lrange(list_key, 0, -1), with_env: with_env)
     end
 
+    BULK_ENV_GET_LUA = <<~LUA
+      local results = {};
+      for i = 1, table.getn(KEYS), 1 do
+        results[i] = { KEYS[i], redis.call('LRANGE', KEYS[i], 0, -1) };
+      end
+      return results;
+    LUA
+
     def bulk_get(message_keys, with_env: true)
       return [] if !message_keys || message_keys.size == 0
-      envs = @redis.mapped_hmget(env_key, *message_keys) if with_env
+      envs = nil
+      if with_env
+        envs = {}
+        @redis.eval(
+          BULK_ENV_GET_LUA,
+          keys: message_keys.map { |k| env_prefix(k) }
+        ).to_h.each do |k, v|
+          next if v.size == 0
+          parsed = v.size == 1 ? JSON.parse(v[0]) : v.map { |e| JSON.parse(e) }
+          envs[env_unprefix(k)] = parsed
+        end
+      end
       messages = @redis.hmget(hash_key, message_keys).map! do |json|
         next if !json || json.size == 0
         message = Message.from_json(json)
-        if with_env
+        if with_env && envs
           env = envs[message.key]
           if !message.env || message.env.size == 0
-            env = env && env.size > 0 ? ::JSON.parse(env) : {}
-            message.env = env
+            message.env = env || {}
           end
         end
         message
@@ -255,20 +272,20 @@ module Logster
     end
 
     def get_env(message_key)
-      json = @redis.hget(env_key, message_key)
-      return if !json || json.size == 0
-      JSON.parse(json)
+      envs = @redis.lrange(env_prefix(message_key), 0, -1)
+      return if !envs || envs.size == 0
+      envs.size == 1 ? JSON.parse(envs[0]) : envs.map { |j| JSON.parse(j) }
     end
 
     def protect(message_key)
-      if message = get(message_key)
+      if message = get(message_key, load_env: false)
         message.protected = true
         update_message(message)
       end
     end
 
     def unprotect(message_key)
-      if message = get(message_key)
+      if message = get(message_key, load_env: false)
         message.protected = false
         update_message(message)
       else
@@ -376,13 +393,11 @@ module Logster
 
     protected
 
-    def clear_solved(count = nil)
-
+    def clear_solved
       ignores = Set.new(@redis.hkeys(solved_key) || [])
 
       if ignores.length > 0
-        start = count ? 0 - count : 0
-        message_keys = @redis.lrange(list_key, start, -1) || []
+        message_keys = @redis.lrange(list_key, 0, -1) || []
 
         bulk_get(message_keys).each do |message|
           unless (ignores & (message.solved_keys || [])).empty?
@@ -397,7 +412,7 @@ module Logster
         removed_keys = []
         while removed_key = @redis.lpop(list_key)
           unless @redis.sismember(protected_key, removed_key)
-            rmsg = get removed_key
+            rmsg = get(removed_key, load_env: false)
             delete(rmsg)
             break
           else
@@ -410,9 +425,9 @@ module Logster
       end
     end
 
-    def update_message(message)
+    def update_message(message, save_env: false)
       @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
-      @redis.hset(env_key, message.key, message.env_json)
+      push_env(message.key, message.env) if save_env
       if message.protected
         @redis.sadd(protected_key, message.key)
       else
@@ -571,7 +586,7 @@ module Logster
     end
 
     def protected_key
-      @saved_key ||= "__LOGSTER__SAVED"
+      @protected_key ||= "__LOGSTER__SAVED"
     end
 
     def grouping_key
@@ -608,22 +623,6 @@ module Logster
       )
     end
 
-    def apply_max_size_limit(message)
-      size = message.to_json(exclude_env: true).bytesize
-      env_size = message.env_json.bytesize
-      max_size = Logster.config.maximum_message_size_bytes
-      if size + env_size > max_size
-        # env is most likely the reason for the large size
-        # truncate it so the overall size is < the limit
-        if Array === message.env
-          # the - 1 at the end ensures the size goes a little bit below the limit
-          truncate_at = (message.env.size.to_f * max_size.to_f / (env_size + size)).to_i - 1
-          truncate_at = 1 if truncate_at < 1
-          message.env = message.env[0...truncate_at]
-        end
-      end
-    end
-
     def register_rate_limit(severities, limit, duration, callback)
       severities = [severities] unless severities.is_a?(Array)
       redis = (@redis_raw_connection && @redis_prefix) ? @redis_raw_connection : @redis
@@ -636,5 +635,24 @@ module Logster
       rate_limits[self.redis_prefix] << rate_limiter
       rate_limiter
     end
+
+    def push_env(message_key, env)
+      prefixed = env_prefix(message_key)
+      env = [env] unless Array === env
+      @redis.lpush(prefixed, env.map(&:to_json).reverse)
+      @redis.ltrim(prefixed, 0, Logster.config.max_env_count_per_message - 1)
+    end
+
+    def delete_env(message_key)
+      @redis.del(env_prefix(message_key))
+    end
+
+    def env_unprefix(key)
+      key.sub(ENV_PREFIX, "")
+    end
+
+    def env_prefix(key)
+      ENV_PREFIX + key
+    end
   end
 end

data/lib/logster/suppression_pattern.rb

@@ -24,7 +24,7 @@ module Logster
     def retro_delete_messages
       keys = []
       grouping_keys = []
-      @store.get_all_messages.each do |message|
+      @store.get_all_messages(with_env: false).each do |message|
         if message =~ self.pattern
           keys << message.key
           grouping_keys << message.grouping_key

data/lib/logster/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Logster
-  VERSION = "2.5.1"
+  VERSION = "2.6.0"
 end

data/logster.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "guard"
   spec.add_development_dependency "guard-minitest"
   spec.add_development_dependency "timecop"
-  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "byebug", "~> 11.0.0"
   spec.add_development_dependency "rubocop", "~> 0.69.0"
   spec.add_development_dependency "rubocop-discourse"
 end

data/test/logster/middleware/test_viewer.rb

@@ -370,4 +370,104 @@ class TestViewer < Minitest::Test
     response = request.get("/logsie/fetch-env/123456abc.json")
     assert_equal(404, response.status)
   end
+
+  def test_solve_group_api_requires_post_request
+    Logster.config.enable_custom_patterns_via_ui = true
+    Logster::GroupingPattern.new(/gotta be post/).save
+    msg = Logster.store.report(
+      Logger::WARN,
+      '',
+      'gotta be post 22',
+      env: { "application_version" => "abc" },
+      backtrace: "aa"
+    )
+    latest = Logster.store.latest
+    assert_equal(1, latest.size)
+    assert_equal(msg.key, latest.first["messages"].first.key)
+    %i[get head options].each do |m|
+      response = request.public_send(m, "/logsie/solve-group", params: { regex: "/gotta be post/" })
+      assert_equal(405, response.status)
+      assert_equal("POST", response.headers["Allow"])
+    end
+    latest = Logster.store.latest
+    assert_equal(1, latest.size)
+    assert_equal(msg.key, latest.first["messages"].first.key)
+  ensure
+    Logster.config.enable_custom_patterns_via_ui = false
+  end
+
+  def test_solve_group_returns_404_when_pattern_doesnt_exist
+    Logster.config.enable_custom_patterns_via_ui = true
+    Logster::GroupingPattern.new(/some pattern/).save
+    msg = Logster.store.report(
+      Logger::WARN,
+      '',
+      'some pattern 22',
+      env: { "application_version" => "abc" },
+      backtrace: "aa"
+    )
+    latest = Logster.store.latest
+    assert_equal(1, latest.size)
+    assert_equal(msg.key, latest.first["messages"].first.key)
+    response = request.post("/logsie/solve-group", params: { regex: "/i dont exist/" })
+    assert_equal(404, response.status)
+    latest = Logster.store.latest
+    assert_equal(1, latest.size)
+    assert_equal(msg.key, latest.first["messages"].first.key)
+  ensure
+    Logster.config.enable_custom_patterns_via_ui = false
+  end
+
+  def test_solving_grouped_messages
+    Logster.config.enable_custom_patterns_via_ui = true
+    backtrace = "a b c d"
+    Logster::GroupingPattern.new(/test pattern/).save
+    msg1 = Logster.store.report(Logger::WARN, '', 'test pattern 1', backtrace: backtrace)
+    msg2 = Logster.store.report(
+      Logger::WARN,
+      '',
+      'test pattern 2',
+      env: { "application_version" => "abc" },
+      backtrace: backtrace
+    )
+    msg3 = Logster.store.report(
+      Logger::WARN,
+      '',
+      'test pattern 3',
+      env: [{ "application_version" => "def" }, { "application_version" => "ghi" }],
+      backtrace: backtrace
+    )
+    group = Logster.store.find_pattern_groups { |p| p == /test pattern/ }.first
+    assert_equal([msg3, msg2, msg1].map(&:key), group.messages_keys)
+
+    latest = Logster.store.latest
+    assert_equal(1, latest.size)
+    assert_equal([msg1, msg2, msg3].map(&:key).sort, latest.first["messages"].map(&:key).sort)
+
+    response = request.post("/logsie/solve-group", params: { regex: "/test pattern/" })
+    group = Logster.store.find_pattern_groups { |p| p == /test pattern/ }.first
+    assert_equal([msg1.key], group.messages_keys)
+    assert_equal(200, response.status)
+
+    latest = Logster.store.latest
+    # msg1 remains cause it doesn't have application_version
+    assert_equal([msg1.key], latest.first["messages"].map(&:key))
+    assert_equal(1, latest.size)
+
+    msg4 = Logster.store.report(Logger::WARN, '', 'test pattern 4', backtrace: backtrace)
+    %w[abc def ghi].each do |version|
+      Logster.store.report(
+        Logger::WARN,
+        '',
+        'test pattern 5',
+        env: { "application_version" => version },
+        backtrace: backtrace
+      )
+    end
+    latest = Logster.store.latest
+    assert_equal([msg1.key, msg4.key].sort, latest.first["messages"].map(&:key).sort)
+    assert_equal(1, latest.size)
+  ensure
+    Logster.config.enable_custom_patterns_via_ui = false
+  end
 end