logster 1.4.0.pre → 2.0.0.pre

data/lib/logster/rails/railtie.rb

@@ -20,7 +20,6 @@ module Logster::Rails
     Logster.logger = ::Rails.logger = config.logger = logger
   end
 
-
   def self.initialize!(app)
     return unless Logster.config.environments.include?(Rails.env.to_sym)
 

data/lib/logster/redis_store.rb

@@ -8,7 +8,7 @@ module Logster
 
     attr_reader :duration, :callback
 
-    def self.clear_all(redis, redis_prefix=nil)
+    def self.clear_all(redis, redis_prefix = nil)
       prefix = key_prefix(redis_prefix)
 
       redis.eval "
@@ -123,7 +123,7 @@ module Logster
     end
 
     def save(message)
-      if keys=message.solved_keys
+      if keys = message.solved_keys
         keys.each do |solved|
           return true if @redis.hget(solved_key, solved)
         end
@@ -144,18 +144,20 @@ module Logster
     def delete(msg)
       @redis.multi do
         @redis.hdel(hash_key, msg.key)
+        @redis.hdel(env_key, msg.key)
         @redis.hdel(grouping_key, msg.grouping_key)
         @redis.lrem(list_key, -1, msg.key)
       end
     end
 
-    def replace_and_bump(message)
+    def replace_and_bump(message, save_env: true)
       # TODO make it atomic
       exists = @redis.hexists(hash_key, message.key)
       return false unless exists
 
       @redis.multi do
-        @redis.hset(hash_key, message.key, message.to_json)
+        @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
+        @redis.hset(env_key, message.key, (message.env || {}).to_json) if save_env
         @redis.lrem(list_key, -1, message.key)
         @redis.rpush(list_key, message.key)
       end
@@ -183,7 +185,7 @@ module Logster
       clear_solved
     end
 
-    def latest(opts={})
+    def latest(opts = {})
       limit = opts[:limit] || 50
       severity = opts[:severity]
       before = opts[:before]
@@ -200,13 +202,12 @@ module Logster
 
       begin
         keys = @redis.lrange(list_key, start, finish) || []
-        break unless keys and keys.count > 0
-        rows = @redis.hmget(hash_key, keys)
+        break unless keys && (keys.count > 0)
+        rows = bulk_get(keys)
 
         temp = []
 
-        rows.each do |s|
-          row = Message.from_json(s)
+        rows.each do |row|
           break if before && before == row.key
           row = nil if severity && !severity.include?(row.severity)
 
@@ -236,10 +237,14 @@ module Logster
       keys = @redis.smembers(protected_key) || []
       if keys.empty?
         @redis.del(hash_key)
+        @redis.del(env_key)
       else
         protected = @redis.mapped_hmget(hash_key, *keys)
+        protected_env = @redis.mapped_hmget(env_key, *keys)
         @redis.del(hash_key)
+        @redis.del(env_key)
         @redis.mapped_hmset(hash_key, protected)
+        @redis.mapped_hmset(env_key, protected_env)
 
         sorted = protected
           .values
@@ -264,15 +269,39 @@ module Logster
       @redis.del(list_key)
       @redis.del(protected_key)
       @redis.del(hash_key)
+      @redis.del(env_key)
       @redis.del(grouping_key)
       @redis.del(solved_key)
     end
 
-    def get(message_key)
+    def get(message_key, load_env: true)
       json = @redis.hget(hash_key, message_key)
       return nil unless json
 
-      Message.from_json(json)
+      message = Message.from_json(json)
+      if load_env
+        message.env = get_env(message_key) || {}
+      end
+      message
+    end
+
+    def bulk_get(message_keys)
+      envs = @redis.hmget(env_key, message_keys)
+      @redis.hmget(hash_key, message_keys).map!.with_index do |json, ind|
+        message = Message.from_json(json)
+        env = envs[ind]
+        if !message.env || message.env.size == 0
+          env = env && env.size > 0 ? ::JSON.parse(env) : {}
+          message.env = env
+        end
+        message
+      end
+    end
+
+    def get_env(message_key)
+      json = @redis.hget(env_key, message_key)
+      return if !json || json.size == 0
+      JSON.parse(json)
     end
 
     def protect(message_key)
@@ -321,10 +350,9 @@ module Logster
 
       if ignores.length > 0
         start = count ? 0 - count : 0
-        message_keys = @redis.lrange(list_key, start, -1 ) || []
+        message_keys = @redis.lrange(list_key, start, -1) || []
 
-        @redis.hmget(hash_key, message_keys).each do |json|
-          message =  Message.from_json(json)
+        bulk_get(message_keys).each do |message|
           unless (ignores & (message.solved_keys || [])).empty?
             delete message
           end
@@ -339,6 +367,7 @@ module Logster
           unless @redis.sismember(protected_key, removed_key)
             rmsg = get removed_key
             @redis.hdel(hash_key, rmsg.key)
+            @redis.hdel(env_key, rmsg.key)
             @redis.hdel(grouping_key, rmsg.grouping_key)
             break
           else
@@ -352,7 +381,8 @@ module Logster
     end
 
     def update_message(message)
-      @redis.hset(hash_key, message.key, message.to_json)
+      @redis.hset(hash_key, message.key, message.to_json(exclude_env: true))
+      @redis.hset(env_key, message.key, (message.env || {}).to_json)
       if message.protected
         @redis.sadd(protected_key, message.key)
       else
@@ -385,7 +415,7 @@ module Logster
       start = -limit
       finish = -1
 
-      return [start,finish] unless before || after
+      return [start, finish] unless before || after
 
       found = nil
       find = before || after
@@ -463,12 +493,12 @@ module Logster
           env_matches?(value, search)
         else
           case search
-            when Regexp
-              value.to_s =~ search
-            when String
-              value.to_s =~ Regexp.new(search, Regexp::IGNORECASE)
+          when Regexp
+            value.to_s =~ search
+          when String
+            value.to_s =~ Regexp.new(search, Regexp::IGNORECASE)
             else
-              false
+            false
           end
         end
       end
@@ -506,6 +536,10 @@ module Logster
       @hash_key ||= "__LOGSTER__MAP"
     end
 
+    def env_key
+      @env_key ||= "__LOGSTER__ENV_MAP"
+    end
+
     def protected_key
       @saved_key ||= "__LOGSTER__SAVED"
     end

data/lib/logster/version.rb

@@ -1,3 +1,3 @@
 module Logster
-  VERSION = "1.4.0.pre"
+  VERSION = "2.0.0.pre"
 end

data/logster.gemspec

@@ -31,4 +31,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "guard-minitest"
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "byebug"
+  spec.add_development_dependency "rubocop", "~> 0.61.1"
 end

data/test/fake_data/generate.rb

@@ -6,5 +6,5 @@ Logster.config.application_version = "ABC123"
 Logster.store = Logster::RedisStore.new
 
 10.times do
-  Logster.store.report(Logger::WARN, "application", "test warning", backtrace: "method1\nmethod2", env: {something: ["hello world", "hello places"], another: {thing: "something else"}})
+  Logster.store.report(Logger::WARN, "application", "test warning", backtrace: "method1\nmethod2", env: { something: ["hello world", "hello places"], another: { thing: "something else" } })
 end

data/test/logster/middleware/test_reporter.rb

@@ -3,7 +3,6 @@ require 'rack'
 require 'logster/redis_store'
 require 'logster/middleware/reporter'
 
-
 class TestReporter < Minitest::Test
 
   def test_logs_errors
@@ -18,4 +17,3 @@ class TestReporter < Minitest::Test
   end
 
 end
-

data/test/logster/middleware/test_viewer.rb

@@ -33,9 +33,9 @@ class TestViewer < Minitest::Test
     assert_nil(viewer.send(:resolve_path, "/logs"))
     assert_nil(viewer.send(:resolve_path, "/admin/logsie"))
     assert_nil(viewer.send(:resolve_path, "/admin/logsie/bla"))
-    assert_equal("/",viewer.send(:resolve_path, "/logsie"))
-    assert_equal("/",viewer.send(:resolve_path, "/logsie/"))
-    assert_equal("/hello/world",viewer.send(:resolve_path, "/logsie/hello/world"))
+    assert_equal("/", viewer.send(:resolve_path, "/logsie"))
+    assert_equal("/", viewer.send(:resolve_path, "/logsie/"))
+    assert_equal("/hello/world", viewer.send(:resolve_path, "/logsie/hello/world"))
   end
 
   def test_search_raceguard_s
@@ -53,7 +53,7 @@ class TestViewer < Minitest::Test
   def test_search_raceguard_f
     response = request.get("/logsie/messages.json?filter=0_1_2_3_4")
     result = JSON.parse(response.body)
-    assert_equal([0,1,2,3,4], result['filter'])
+    assert_equal([0, 1, 2, 3, 4], result['filter'])
   end
 
   def test_regex_parse

data/test/logster/test_base_store.rb

@@ -53,14 +53,14 @@ class TestBaseStore < Minitest::Test
 
   def test_ignore_pattern_basic
     @store.ignore = [
-        Logster::IgnorePattern.new(nil, {username: 'CausingErrors'})
+        Logster::IgnorePattern.new(nil, username: 'CausingErrors')
     ]
     @store.report(Logger::WARN, "test", "Foobar") #
-    @store.report(Logger::WARN, "test", "Foobar", { env: { username: 'CausingErrors' }})
+    @store.report(Logger::WARN, "test", "Foobar", env: { username: 'CausingErrors' })
     @store.report(Logger::WARN, "test", "Foobar", env: nil)
-    @store.report(Logger::WARN, "test", "Something Else", { env: { username: 'CausingErrors' }})
-    @store.report(Logger::WARN, "test", "Something Else", { env: { 'username' => 'CausingErrors' }})
-    @store.report(Logger::WARN, "test", "Something Else", { env: { username: 'GoodPerson' }}) #
+    @store.report(Logger::WARN, "test", "Something Else", env: { username: 'CausingErrors' })
+    @store.report(Logger::WARN, "test", "Something Else", env: { 'username' => 'CausingErrors' })
+    @store.report(Logger::WARN, "test", "Something Else", env: { username: 'GoodPerson' }) #
     @store.report(Logger::WARN, "test", "Can't verify CSRF token authenticity") #
 
     assert_equal(4, @store.count)
@@ -69,10 +69,10 @@ class TestBaseStore < Minitest::Test
   def test_ignore_pattern_real
     @store.ignore = [
         /^ActionController::RoutingError \(No route matches/,
-        Logster::IgnorePattern.new("Can't verify CSRF token authenticity", { REQUEST_URI: /\/trackback\/$/ })
+        Logster::IgnorePattern.new("Can't verify CSRF token authenticity", REQUEST_URI: /\/trackback\/$/)
     ]
     # blocked
-    @store.report(Logger::WARN, "whatever", "Can't verify CSRF token authenticity", {
+    @store.report(Logger::WARN, "whatever", "Can't verify CSRF token authenticity",
         env: {
             HTTP_HOST: 'meta.discourse.org',
             REQUEST_URI: '/t/use-more-standard-smiley-codes-instead-of-smile/1822/trackback/',
@@ -85,9 +85,9 @@ class TestBaseStore < Minitest::Test
                 blog_name: 'get free spam for cheap'
             }
         }
-    })
+    )
     # logged
-    @store.report(Logger::WARN, "whatever", "Can't verify CSRF token authenticity", {
+    @store.report(Logger::WARN, "whatever", "Can't verify CSRF token authenticity",
         env: {
             HTTP_HOST: 'meta.discourse.org',
             REQUEST_URI: '/session',
@@ -99,12 +99,12 @@ class TestBaseStore < Minitest::Test
                 form_authenticity_token: 'incorrect'
             }
         }
-    })
+    )
     assert_equal(1, @store.count)
   end
 
   def test_timestamp
-    time = Time.now - 24*60*60
+    time = Time.now - 24 * 60 * 60
     message = @store.report(Logger::WARN, "test", "B", timestamp: time)
 
     assert_equal(time, message.timestamp)
@@ -122,20 +122,20 @@ class TestBaseStore < Minitest::Test
     # Backtrace can be passed via backtrace param or env
     message = @store.report(Logger::WARN, "test", "A", backtrace: error.backtrace)
     assert_equal(error.backtrace.join("\n"), message.backtrace)
-    message = @store.report(Logger::WARN, "test", "B", env: {backtrace: error.backtrace})
+    message = @store.report(Logger::WARN, "test", "B", env: { backtrace: error.backtrace })
     assert_equal(error.backtrace.join("\n"), message.backtrace)
 
     # Via env takes priority
-    message = @store.report(Logger::WARN, "test", "C", backtrace: "Garbage", env: {backtrace: error.backtrace})
+    message = @store.report(Logger::WARN, "test", "C", backtrace: "Garbage", env: { backtrace: error.backtrace })
     assert_equal(error.backtrace.join("\n"), message.backtrace)
 
     # Backtrace is always a string
     # Cannot do an equal assert here, because it uses `caller` when not provided
     message = @store.report(Logger::WARN, "test", "D", backtrace: nil)
     assert_kind_of(String, message.backtrace)
-    message = @store.report(Logger::WARN, "test", "E", env: {backtrace: nil})
+    message = @store.report(Logger::WARN, "test", "E", env: { backtrace: nil })
     assert_kind_of(String, message.backtrace)
-    message = @store.report(Logger::WARN, "test", "F", backtrace: nil, env: {backtrace: nil})
+    message = @store.report(Logger::WARN, "test", "F", backtrace: nil, env: { backtrace: nil })
     assert_kind_of(String, message.backtrace)
     message = @store.report(Logger::WARN, "test", "G")
     assert_kind_of(String, message.backtrace)

data/test/logster/test_ignore_pattern.rb

@@ -17,25 +17,25 @@ class TestIgnorePattern < Minitest::Test
 
   def test_env_pattern
     msg = Logster::Message.new(Logger::WARN, "test", "my error")
-    msg.env = {"frogs" => "are big"}
+    msg.env = { "frogs" => "are big" }
 
-    pattern = Logster::IgnorePattern.new(nil, {frogs: "big"})
+    pattern = Logster::IgnorePattern.new(nil, frogs: "big")
 
     assert pattern.matches? msg
 
-    msg.env = {legs: nil}
+    msg.env = { legs: nil }
     assert !(pattern.matches? msg)
 
-    msg.env = {legs: 3}
+    msg.env = { legs: 3 }
     assert !(pattern.matches? msg)
 
-    msg.env = {frogs: "small"}
+    msg.env = { frogs: "small" }
     assert !pattern.matches?(msg)
 
     pattern = Logster::IgnorePattern.new(nil, "small")
     assert pattern.matches? msg
 
-    msg.env = {frogs: "big"}
+    msg.env = { frogs: "big" }
     assert !(pattern.matches? msg)
   end
 end

data/test/logster/test_logger.rb

@@ -26,19 +26,19 @@ class TestLogger < Minitest::Test
   def test_per_thread_override
     @logger.override_level = 2
 
-    @logger.add(0, "test", "prog", backtrace: "backtrace", env: {a: "x"})
+    @logger.add(0, "test", "prog", backtrace: "backtrace", env: { a: "x" })
     Thread.new do
-      @logger.add(0, "test", "prog", backtrace: "backtrace", env: {a: "x"})
+      @logger.add(0, "test", "prog", backtrace: "backtrace", env: { a: "x" })
     end.join
 
     @logger.override_level = nil
-    @logger.add(0, "test", "prog", backtrace: "backtrace", env: {a: "x"})
+    @logger.add(0, "test", "prog", backtrace: "backtrace", env: { a: "x" })
 
     assert_equal 2, @store.calls.length
   end
 
   def test_backtrace
-    @logger.add(0, "test", "prog", backtrace: "backtrace", env: {a: "x"})
+    @logger.add(0, "test", "prog", backtrace: "backtrace", env: { a: "x" })
     assert_equal "backtrace", @store.calls[0][3][:backtrace]
   end
 
@@ -54,20 +54,26 @@ class TestLogger < Minitest::Test
     @other_store = TestStore.new
     @logger.chain(Logster::Logger.new(@other_store))
 
-    @logger.add(0, "test", "prog", backtrace: "backtrace", env: {a: "x"})
+    @logger.add(0, "test", "prog", backtrace: "backtrace", env: { a: "x" })
 
     [@store, @other_store].each do |store|
       assert_equal "backtrace", store.calls[0][3][:backtrace]
     end
   end
 
+  def test_progname_parameter
+    @logger.add(0, "test")
+    progname = @store.calls[0][1]
+    assert_nil progname
+  end
+
   class PlayLogger
     attr_accessor :skip_store
     def initialize(tester)
       @tester = tester
     end
 
-    def add(s,m,p,&block)
+    def add(s, m, p, &block)
       @tester.assert(skip_store)
     end
   end

data/test/logster/test_message.rb

@@ -14,14 +14,12 @@ class TestMessage < MiniTest::Test
 
     msg1.merge_similar_message(msg2)
 
-    msg1 = Logster::Message.from_json(msg1.to_json)
-
     assert_equal(20, msg1.timestamp)
     assert_equal(10, msg1.first_timestamp)
 
     assert Array === msg1.env
     assert_equal(msg1.env.size, 2)
-    assert({ "a" => "1", "b" => "2" } <= msg1.env[0])
+    assert({ a: "1", b: "2" } <= msg1.env[0])
     assert({ "a" => "2", "c" => "3" } <= msg1.env[1])
   end
 
@@ -32,14 +30,13 @@ class TestMessage < MiniTest::Test
     msg2 = Logster::Message.new(0, '', 'test', 20)
     msg2.env = [{ e: "ee", f: "ff" }, { g: "gg", h: "hh" }]
 
-    msg1.merge_similar_message(msg2) 
-    msg1 = Logster::Message.from_json(msg1.to_json)
+    msg1.merge_similar_message(msg2)
 
     # new env should be an array, but it should never have
     # another array of envs within itself (hence flatten(1))
     assert_equal(msg1.env.size, 4)
-    assert_equal(msg1.env.map(&:keys).flatten(1), %w{a b c d e f g h})
-    assert_equal(msg1.env.map(&:values).flatten(1), %w{aa bb cc dd ee ff gg hh})
+    assert_equal(msg1.env.map(&:keys).flatten(1).map(&:to_s), %w{a b c d e f g h})
+    assert_equal(msg1.env.map(&:values).flatten(1).map(&:to_s), %w{aa bb cc dd ee ff gg hh})
   end
 
   def test_merge_messages_one_with_array_envs
@@ -49,12 +46,11 @@ class TestMessage < MiniTest::Test
     msg2 = Logster::Message.new(0, '', 'test', 20)
     msg2.env = { e: "ee", f: "ff" }
 
-    msg1.merge_similar_message(msg2) 
-    msg1 = Logster::Message.from_json(msg1.to_json)
+    msg1.merge_similar_message(msg2)
 
     assert_equal(msg1.env.size, 3)
-    assert_equal(msg1.env.map(&:keys).flatten(1), %w{a b c d e f})
-    assert_equal(msg1.env.map(&:values).flatten(1), %w{aa bb cc dd ee ff})
+    assert_equal(msg1.env.map(&:keys).flatten(1).map(&:to_s), %w{a b c d e f})
+    assert_equal(msg1.env.map(&:values).flatten(1).map(&:to_s), %w{aa bb cc dd ee ff})
   end
 
   def test_adds_application_version
@@ -75,8 +71,8 @@ class TestMessage < MiniTest::Test
 
     assert_equal(msg1.grouping_key, msg2.grouping_key)
 
-    msg1.merge_similar_message(msg2)
-    msg1 = Logster::Message.from_json(msg1.to_json)
+    save_env = msg1.merge_similar_message(msg2)
+    assert(save_env)
     assert_equal(msg1.count, 15 + 13)
   end
 
@@ -85,9 +81,39 @@ class TestMessage < MiniTest::Test
     hash = { "custom_key" => "key" }
     msg.populate_from_env([hash])
 
-    msg = Logster::Message.from_json(msg.to_json)
     assert Array === msg.env
     assert_equal(msg.env.size, 1)
     assert hash <= msg.env[0]
   end
+
+  def test_ensure_env_samples_dont_exceed_50
+    msg1 = Logster::Message.new(0, '', 'test', 10, count: 50)
+    msg1.env = [{ a: 1 }]
+    msg2 = Logster::Message.new(0, '', 'test', 20, count: 13)
+    msg2.env = { b: 2 }
+
+    assert_equal(msg1.grouping_key, msg2.grouping_key)
+
+    save_env = msg1.merge_similar_message(msg2)
+
+    refute(save_env)
+    assert_equal(63, msg1.count) # update count
+    assert_equal([{ a: 1 }], msg1.env) # but don't merge msg2's env into msg1's
+  end
+
+  def test_message_to_h_respects_params
+    msg = Logster::Message.new(0, "", "test")
+    test_hash = { test_key: "this is a test" }
+    msg.env = test_hash
+    assert_equal(test_hash, msg.to_h[:env])
+    assert_nil(msg.to_h(exclude_env: true)[:env])
+  end
+
+  def test_message_to_json_respects_params
+    msg = Logster::Message.new(0, "", "test")
+    test_hash = { test_key: "this is a test" }
+    msg.env = test_hash
+    assert_includes(msg.to_json, test_hash.to_json)
+    refute_includes(msg.to_json(exclude_env: true), test_hash.to_json)
+  end
 end