logster 2.5.1 → 2.6.0

data/test/logster/test_base_store.rb

@@ -146,4 +146,20 @@ class TestBaseStore < Minitest::Test
     message = @store.report(Logger::WARN, "test", "H", backtrace: ["Foo", "Bar"])
     assert_equal("Foo\nBar", message.backtrace)
   end
+
+  def test_chained_loggers_dont_have_superfluous_frames_in_backtrace
+    logger = Logster::Logger.new(@store)
+    other_store = Logster::TestStore.new
+    other_logger = Logster::Logger.new(other_store)
+    logger.chain(other_logger)
+    logger.warn("this is warning")
+    [@store, other_store].each do |store|
+      message = store.reported.first
+      assert_equal("this is warning", message.message)
+      # the first line in the backtrace should be the method that
+      # called the warn/info/error etc. method.
+      # in this case the first line should be this test method
+      assert_includes(message.backtrace.lines.first, __method__.to_s)
+    end
+  end
 end

data/test/logster/test_defer_logger.rb

@@ -29,7 +29,7 @@ class TestDeferLogger < Minitest::Test
     # Otherwise we'd only get a partial backtrace from
     # the point the new thread was spawned
     backtrace = @store.calls.first[3][:backtrace]
-    assert_instance_of(Array, backtrace)
+    assert_includes(backtrace.lines.first, __method__.to_s)
 
     assert_equal(0, queue.size)
   end

data/test/logster/test_message.rb

@@ -11,32 +11,38 @@ class TestMessage < MiniTest::Test
 
     msg2 = Logster::Message.new(0, '', 'test', 20)
     msg2.populate_from_env(a: "2", c: "3")
-
     assert_equal(msg2.grouping_key, msg1.grouping_key)
-
     msg1.merge_similar_message(msg2)
 
     assert_equal(20, msg1.timestamp)
     assert_equal(10, msg1.first_timestamp)
-
-    assert Array === msg1.env
-    assert_equal(msg1.env.size, 2)
-    assert({ a: "1", b: "2" } <= msg1.env[0])
-    assert({ "a" => "2", "c" => "3" } <= msg1.env[1])
+    assert_equal(msg1.env_buffer, [msg2.env])
   end
 
-  def test_merge_adds_timestamp_to_env
-    time1 = Time.new(2010, 1, 1, 1, 1).to_i
-    msg1 = Logster::Message.new(0, '', 'test', time1)
-    msg1.env = { a: "aa", b: "bb" }
-
-    time2 = Time.new(2011, 1, 1, 1, 1).to_i
-    msg2 = Logster::Message.new(0, '', 'test', time2)
-    msg2.env = { e: "ee", f: "ff" }
-
-    msg1.merge_similar_message(msg2)
-    assert_equal(time1, msg1.env[0]["time"])
-    assert_equal(time2, msg1.env[1]["time"])
+  def test_populate_from_env_will_add_time_to_env_unless_it_already_exists
+    t = (Time.new.to_f * 1000).to_i
+    msg = Logster::Message.new(0, '', 'test', t)
+    msg.populate_from_env({})
+    assert_equal(t, msg.env["time"])
+
+    msg = Logster::Message.new(0, '', 'test', t)
+    msg.populate_from_env(time: 5)
+    assert_nil(msg.env["time"])
+    assert_equal(5, msg.env[:time])
+
+    msg = Logster::Message.new(0, '', 'test', t)
+    msg.populate_from_env("time" => 6)
+    assert_equal(6, msg.env["time"])
+    assert_nil(msg.env[:time])
+
+    msg = Logster::Message.new(0, '', 'test', t)
+    msg.populate_from_env([{ "time" => 6 }, { "time" => 8 }])
+    assert_equal([6, 8], msg.env.map { |e| e["time"] })
+    assert_equal([nil, nil], msg.env.map { |e| e[:time] })
+
+    msg = Logster::Message.new(0, '', 'test', t)
+    msg.populate_from_env([{ dsd: 6 }, { dsd: 8 }])
+    assert_equal([t, t], msg.env.map { |e| e["time"] })
   end
 
   def test_merge_messages_both_with_array_envs
@@ -47,26 +53,18 @@ class TestMessage < MiniTest::Test
     msg2.env = [{ e: "ee", f: "ff" }, { g: "gg", h: "hh" }]
 
     msg1.merge_similar_message(msg2)
-
-    # new env should be an array, but it should never have
-    # another array of envs within itself (hence flatten(1))
-    assert_equal(msg1.env.size, 4)
-    assert_equal(msg1.env.map(&:keys).flatten(1).map(&:to_s), %w{a b c d e f g h})
-    assert_equal(msg1.env.map(&:values).flatten(1).map(&:to_s), %w{aa bb cc dd ee ff gg hh})
+    assert_equal(msg2.env, msg1.env_buffer)
   end
 
   def test_merge_messages_one_with_array_envs
     msg1 = Logster::Message.new(0, '', 'test', 10)
-    msg1.env = [{ a: "aa", b: "bb" }, { c: "cc", d: "dd" }]
+    msg1.env = { e: "ee", f: "ff" }
 
     msg2 = Logster::Message.new(0, '', 'test', 20)
-    msg2.env = { e: "ee", f: "ff" }
+    msg2.env = [{ a: "aa", b: "bb" }, { c: "cc", d: "dd" }]
 
     msg1.merge_similar_message(msg2)
-
-    assert_equal(msg1.env.size, 3)
-    assert_equal(msg1.env.map(&:keys).flatten(1).map(&:to_s), %w{a b c d e f time})
-    assert_equal(msg1.env.map(&:values).flatten(1).map(&:to_s), %w{aa bb cc dd ee ff 20})
+    assert_equal(msg2.env, msg1.env_buffer)
   end
 
   def test_adds_application_version
@@ -87,8 +85,7 @@ class TestMessage < MiniTest::Test
 
     assert_equal(msg1.grouping_key, msg2.grouping_key)
 
-    save_env = msg1.merge_similar_message(msg2)
-    assert(save_env)
+    msg1.merge_similar_message(msg2)
     assert_equal(msg1.count, 15 + 13)
   end
 
@@ -120,19 +117,17 @@ class TestMessage < MiniTest::Test
     assert hash <= msg.env[0]
   end
 
-  def test_ensure_env_samples_dont_exceed_50
+  def test_merging_envs_add_new_envs_to_buffer
     msg1 = Logster::Message.new(0, '', 'test', 10, count: 50)
-    msg1.env = [{ a: 1 }]
+    msg1.env = 50.times.map { |n| { a: n } }
     msg2 = Logster::Message.new(0, '', 'test', 20, count: 13)
-    msg2.env = { b: 2 }
+    msg2.env = 13.times.map { |n| { b: n } }
 
     assert_equal(msg1.grouping_key, msg2.grouping_key)
 
-    save_env = msg1.merge_similar_message(msg2)
-
-    refute(save_env)
+    msg1.merge_similar_message(msg2)
     assert_equal(63, msg1.count) # update count
-    assert_equal([{ a: 1 }], msg1.env) # but don't merge msg2's env into msg1's
+    assert_equal(msg2.env, msg1.env_buffer)
   end
 
   def test_message_to_h_respects_params
@@ -157,21 +152,114 @@ class TestMessage < MiniTest::Test
     assert_equal(600 + 3, msg.message.size)
   end
 
-  def test_env_is_not_merged_into_similar_message_if_size_will_be_too_large
-    default = Logster.config.maximum_message_size_bytes
-    Logster.config.maximum_message_size_bytes = 1000
-    message = Logster::Message.new(Logger::INFO, "test", "message", count: 13)
-    env = [{ key1: "this is my first key", key2: "this is my second key" }] * 13
-    message.env = env
+  def test_drop_redundant_envs
+    message = Logster::Message.new(Logger::WARN, '', 'message')
+    message.env = [{ a: 4 }] * 10
+    assert_equal(10, message.env.size)
+    message.drop_redundant_envs(5)
+    assert_equal(5, message.env.size)
+
+    env = { f: 5, g: 4 }
+    message.env = env.dup
+    message.drop_redundant_envs(1)
+    assert_equal(env, message.env)
+  end
 
-    message2 = Logster::Message.new(Logger::INFO, "test", "message")
-    message2.env = env.first
-    message.merge_similar_message(message2)
+  def test_apply_env_size_limit_keeps_as_many_keys_as_possible
+    message = Logster::Message.new(Logger::WARN, '', 'message', 1)
+    env = { a: 1, bb: 22, ccc: 333 }
+    message.env = env.dup
+    message.apply_env_size_limit(24)
+    assert_operator(message.env.to_json.bytesize, :<=, 24)
+    assert_equal({ a: 1, bb: 22 }.to_json.bytesize, message.env.to_json.bytesize)
+
+    message.env = [env.dup] * 5
+    message.apply_env_size_limit(24)
+    assert_equal(5, message.env.size)
+    message.env.each do |e|
+      assert_operator(e.to_json.bytesize, :<=, 24)
+      assert_equal({ a: 1, bb: 22 }.to_json.bytesize, e.to_json.bytesize)
+    end
+
+    message.env = env.dup
+    message.apply_env_size_limit(25)
+    assert_operator(message.env.to_json.bytesize, :<=, 25)
+    assert_equal({ a: 1, bb: 22, ccc: 333 }.to_json.bytesize, message.env.to_json.bytesize)
+  end
 
-    # env isn't merged, but count is incremented
-    assert_equal(13, message.env.size)
-    assert_equal(14, message.count)
-  ensure
-    Logster.config.maximum_message_size_bytes = default
+  def test_apply_message_size_limit_removes_gems_dir_from_backtrace_to_keep_total_message_size_below_limit
+    backtrace = <<~TEXT
+      /var/www/discourse/vendor/bundle/ruby/2.6.0/gems/rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:220:in `with_connection'
+      /var/www/discourse/vendor/bundle/ruby/2.6.0/gems/rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:60:in `with_connection'
+      /var/www/discourse/lib/scheduler/defer.rb:89:in `do_work'
+      /var/www/discourse/lib/scheduler/defer.rb:79:in `block (2 levels) in start_thread'
+    TEXT
+    without_gems_dir = <<~TEXT
+      rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:220:in `with_connection'
+      rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:60:in `with_connection'
+      /var/www/discourse/lib/scheduler/defer.rb:89:in `do_work'
+      /var/www/discourse/lib/scheduler/defer.rb:79:in `block (2 levels) in start_thread'
+    TEXT
+    gems_dir = "/var/www/discourse/vendor/bundle/ruby/2.6.0/gems/"
+    message = Logster::Message.new(Logger::WARN, '', 'message', 1)
+
+    message.backtrace = backtrace.dup
+    assert_operator(message.to_json(exclude_env: true).bytesize, :>=, 500)
+    message.apply_message_size_limit(500, gems_dir: gems_dir)
+    assert_operator(message.to_json(exclude_env: true).bytesize, :<=, 500)
+    assert_equal(without_gems_dir.strip, message.backtrace.strip)
+  end
+
+  def test_apply_message_size_limit_removes_lines_from_backtrace_to_keep_total_size_below_limit
+    backtrace = <<~TEXT
+      rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:220:in `with_connection'
+      rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:60:in `with_connection'
+      /var/www/discourse/lib/scheduler/defer.rb:89:in `do_work'
+      /var/www/discourse/lib/scheduler/defer.rb:79:in `block (2 levels) in start_thread'
+    TEXT
+
+    expected = <<~TEXT
+      rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:220:in `with_connection'
+      rails_multisite-2.0.7/lib/rails_multisite/connection_management.rb:60:in `with_connection'
+      /var/www/discourse
+    TEXT
+    message = Logster::Message.new(Logger::WARN, '', 'message', 1)
+    message.backtrace = backtrace.dup
+    assert_operator(message.to_json(exclude_env: true).bytesize, :>=, 350)
+    message.apply_message_size_limit(350)
+    assert_operator(message.to_json(exclude_env: true).bytesize, :<=, 350)
+    assert_equal(expected.strip, message.backtrace.strip)
+  end
+
+  def test_truncate_backtrace_shouldnt_corrupt_backtrace_when_it_contains_multibytes_characters
+    backtrace = "aहa"
+    message = Logster::Message.new(Logger::WARN, '', 'message', 1)
+    message.backtrace = backtrace.dup
+    message.truncate_backtrace(3)
+    assert_equal("a", message.backtrace)
+
+    message.backtrace = backtrace.dup
+    message.truncate_backtrace(4)
+    assert_equal("aह", message.backtrace)
+
+    message.backtrace = backtrace.dup
+    message.truncate_backtrace(5)
+    assert_equal(backtrace, message.backtrace)
+  end
+
+  def test_apply_message_size_limit_doesnt_remove_backtrace_entirely
+    message = Logster::Message.new(Logger::WARN, '', 'message', 1)
+    message.backtrace = "a" * 1000
+    assert_operator(message.to_json(exclude_env: true).bytesize, :>=, 500)
+    message.apply_message_size_limit(500)
+    assert_operator(message.to_json(exclude_env: true).bytesize, :<=, 500)
+    assert_equal(("a" * 354).size, message.backtrace.size)
+  end
+
+  def test_apply_message_size_limit_doesnt_hang_forever_and_doesnt_remove_backtrace_entirely
+    message = Logster::Message.new(Logger::WARN, '', 'message', 1)
+    message.backtrace = "aa" * 100
+    message.apply_message_size_limit(10)
+    assert_equal(("aa" * 100).size, message.backtrace.size)
   end
 end

data/test/logster/test_redis_store.rb

@@ -223,10 +223,11 @@ class TestRedisStore < Minitest::Test
       msg = messages[n]
       assert_equal("test_#{n}", msg.message)
       if n == 0
-        assert_equal(Logster::Message.default_env, msg.env)
+        assert_equal(Logster::Message.default_env.merge("time" => msg.timestamp), msg.env)
       else
         assert({ "test_#{n}" => "envsss" } <= msg.env)
       end
+      assert_equal(msg.timestamp, msg.env["time"])
     end
   end
 
@@ -241,43 +242,50 @@ class TestRedisStore < Minitest::Test
     old_env = { "old_env" => "old value" }
     message = @store.report(Logger::WARN, "test", "A", env: old_env)
 
-    unsaved_env = { "unsaved_env" => "lost value" }
-    message.env = unsaved_env
+    extra_env = { "saved_env" => "saved value!" }
+    similar = @store.report(Logger::WARN, 'test', 'A', env: extra_env)
+    message.merge_similar_message(similar)
 
-    @store.replace_and_bump(message, save_env: false)
+    @store.replace_and_bump(message)
 
     message = @store.get(message.key)
-    assert(old_env <= message.env)
-    refute(unsaved_env <= message.env)
-
-    saved_env = { "saved_env" => "saved value!" }
-    message.env = saved_env
-
-    @store.replace_and_bump(message)
+    assert(extra_env <= message.env[0])
+    assert(old_env <= message.env[1])
+  end
 
+  def test_ensure_env_doesnt_exceed_50_item
+    Logster.config.allow_grouping = true
+    message = nil
+    52.times do |n|
+      message = @store.report(Logger::WARN, "", "mssage", env: { a: n })
+    end
     message = @store.get(message.key)
-    assert(saved_env == message.env)
+    assert_equal(52, message.count)
+    assert_equal(50, message.env.size)
+    assert_equal((2..51).to_a, message.env.map { |e| e[:a] || e["a"] }.sort)
+  ensure
+    Logster.config.allow_grouping = false
   end
 
   def test_merging_performance
     Logster.config.allow_grouping = true
     backtrace = "fake backtrace"
-    env = { "some_env" => "some env" }
-    another_env = { "another_env" => "more env" }
-    yet_another_env = { "moaar_env" => "more env" }
-
-    @store.report(Logger::WARN, "", "title", backtrace: backtrace, env: env, count: 49)
+    env = [{ "some_env" => "some env" }] * 50
+    new_env = { "some_key" => "1234442" }
 
-    message = @store.report(Logger::WARN, "", "title", backtrace: backtrace, env: another_env)
-    assert_instance_of(Array, message.env)
-    assert_equal(2, message.env.size)
-    assert(env <= message.env[0])
-    assert(another_env <= message.env[1])
+    @store.report(Logger::WARN, "", "title", backtrace: backtrace, env: env, count: 50)
 
-    message = @store.report(Logger::WARN, "", "title", backtrace: backtrace, env: yet_another_env)
-    # we don't need to load env from redis cause we don't
-    # need to merge new env samples if count is 50 or more
+    message = @store.report(Logger::WARN, "", "title", backtrace: backtrace, env: new_env)
+    # env is nil cause we don't need to fetch it from redis
+    # we just send the newly added envs to redis and it'll
+    # take care of prepending them to the existing envs
     assert_nil(message.env)
+
+    message = @store.get(message.key)
+    assert_instance_of(Array, message.env)
+    assert_equal(50, message.env.size)
+    assert_equal(51, message.count)
+    assert(new_env <= message.env[0])
   ensure
     Logster.config.allow_grouping = false
   end
@@ -548,7 +556,7 @@ class TestRedisStore < Minitest::Test
       # message2 shouldn't vanish even if
       # its env matches an ignore pattern
       # however it should be merged with message1
-      assert_equal("business17", message.env[1]["cluster"])
+      assert_equal("business17", message.env[0]["cluster"])
     ensure
       # reset so it doesn't affect other tests
       @store.ignore = nil
@@ -833,19 +841,46 @@ class TestRedisStore < Minitest::Test
     end
   end
 
-  def test_store_trims_too_big_envs
-    default = Logster.config.maximum_message_size_bytes
-    Logster.config.maximum_message_size_bytes = 1000
-    message = Logster::Message.new(Logger::INFO, "test", "message")
-    env = [{ key1: "this is my first key", key2: "this is my second key" }] * 40
-    message.env = env
-    @store.save(message)
-    trimmed_message = @store.latest.first
-    assert_equal(13, trimmed_message.env.size)
-    size = message.to_json(exclude_env: true).bytesize + message.env_json.bytesize
-    assert_operator(1000, :>, size)
-  ensure
-    Logster.config.maximum_message_size_bytes = default
+  def test_ensure_messages_meet_config_size_limits_when_messages_are_saved
+    config_reset(
+      maximum_message_size_bytes: 300,
+      max_env_bytes: 30,
+      max_env_count_per_message: 5
+    ) do
+      env = [{ aaa: 111, bbb: 222, ccc: 333, ddd: 444 }] * 7
+      message = @store.report(Logger::WARN, '', 'test', backtrace: "aa\n" * 100, env: env.dup, timestamp: 777)
+      message = @store.get(message.key)
+      assert_operator(message.to_json(exclude_env: true).bytesize, :<, 300)
+      assert_equal(5, message.env.size)
+      message.env.each do |e|
+        assert_operator(e.to_json.bytesize, :<=, 30)
+        assert_equal({ "aaa" => 111, "time" => 777 }, e)
+      end
+    end
+  end
+
+  def test_ensure_messages_meet_config_size_limits_when_merged_together
+
+    config_reset(
+      max_env_bytes: 30,
+      max_env_count_per_message: 5,
+      allow_grouping: true
+    ) do
+      env = [{ a: 1, aa: 22, aaa: 333, aaaa: 4444 }] * 3
+      env_2 = [{ b: 1, bb: 22, bbb: 333, bbbb: 4444 }] * 3
+      @store.report(Logger::WARN, '', 'test', backtrace: "aa\n" * 100, env: env.dup, timestamp: 777)
+      message = @store.report(Logger::WARN, '', 'test', backtrace: "aa\n" * 100, env: env_2.dup, timestamp: 777)
+      message = @store.get(message.key)
+      assert_equal(5, message.env.size)
+      message.env.first(3).each do |e|
+        assert_operator(e.to_json.bytesize, :<=, 30)
+        assert_equal({ "b" => 1, "bb" => 22, "time" => 777 }, e)
+      end
+      message.env.last(2).each do |e|
+        assert_operator(e.to_json.bytesize, :<=, 30)
+        assert_equal({ "a" => 1, "aa" => 22, "time" => 777 }, e)
+      end
+    end
   end
 
   def test_custom_grouping_patterns
@@ -997,8 +1032,33 @@ class TestRedisStore < Minitest::Test
     Logster::Group.remove_instance_variable(:@max_size)
   end
 
+  def test_truncated_messages_when_they_are_similar_can_still_be_merged
+    config_reset(allow_grouping: true) do
+      backtrace = "a" * Logster.config.maximum_message_size_bytes
+      title = "sasasas"
+      msg = @store.report(Logger::WARN, '', title, backtrace: backtrace.dup)
+      msg2 = @store.report(Logger::WARN, '', title, backtrace: backtrace.dup)
+      assert_equal(msg.key, msg2.key)
+      assert_operator(msg.to_json(exclude_env: true).bytesize, :<=, Logster.config.maximum_message_size_bytes)
+      assert_operator(msg.backtrace.size, :<, backtrace.size)
+    end
+  end
+
   private
 
+  def config_reset(configs)
+    defaults = {}
+    configs.each do |k, v|
+      defaults[k] = Logster.config.public_send(k)
+      Logster.config.public_send("#{k}=", v)
+    end
+    yield
+  ensure
+    defaults.each do |k, v|
+      Logster.config.public_send("#{k}=", v)
+    end
+  end
+
   def reset_redis
     @store.clear_all
   end