logster 1.0.1 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56302ee5e8c4578baa96303208225a8802e9413a
-  data.tar.gz: 41aed485e25ed9efd4e57aac6b57ee83a921ef2f
+  metadata.gz: 10338f67f3b0de4bc895fab64a21addb21987b02
+  data.tar.gz: 3d4b900cba51c32ccbf7aad9b15843c3f2cb5a97
 SHA512:
-  metadata.gz: 9a8ea2cf4353b00fb3f60697f094f5f2b9a25d08032fbb160d6639a7462c6dd77624215b20964b4cfcb961e0eb2173815d2595a2db62ee01f5bb56af4f2f9a9b
-  data.tar.gz: 90535bd3068bc8124e01f5ceb0f771080acb1a353b8a6a047227e00ba267caf624e7f1355ab2865bbc0cb2414c3205e5bcf4fa647dd2583f8abc666cefa2aab9
+  metadata.gz: 6ff41f127c57daf3404518e1311d980887b25366f20b7330d73b8b0814e8d5ca9ceed7ad5df4d33cc270806d9eb7a1c3f81787ba805566102dfd06eda42a9186
+  data.tar.gz: 9f62de9005f735c84c121ac9a7dc09259afc42dc8c032a162b7d0770bb96b62f8045a0e679febbc9d8a74af8fb2488fa27e08cb077a3aea1bce185a252929075

data/.travis.yml

@@ -0,0 +1,18 @@
+language: ruby
+
+matrix:
+  fast_finish: true
+
+rvm:
+  - 2.0.0
+  - 2.1
+  - 2.2
+  - 2.3.0
+
+services:
+  - redis-server
+
+before_install:
+  - gem install bundler
+
+sudo: false

data/README.md

@@ -1,10 +1,10 @@
-![logster logo](https://raw.githubusercontent.com/discourse/logster/master/assets/images/logo-logster-cropped-small.png)
+![logster logo](https://raw.githubusercontent.com/discourse/logster/master/website/images/logo-logster-cropped-small.png)
 
 Logster is an embedded Ruby "exception reporting service" admins can view on live websites, at `http://example.com/logs`
 
 ## Interface
 
-![Screenshot](https://raw.githubusercontent.com/discourse/logster/master/assets/images/logster-screenshot.png)
+![Screenshot](https://raw.githubusercontent.com/discourse/logster/master/website/images/logster-screenshot.png)
 
 Play with a live demo at [logster.info/logs](http://logster.info/logs).
 
@@ -35,6 +35,23 @@ To run logster in other environments, in `config/application.rb`
 Logster.set_environments([:development, :staging, :production])
 ```
 
+### Tracking Error Rate
+Logster allows you to register a callback when the rate of errors has exceeded
+a given limit.
+
+Tracking buckets available are one minute and an hour.
+
+Example:
+```
+Logster.register_rate_limit_per_minute(Logger::WARN, 60) do |rate|
+  puts "O no! The error rate is now #{rate} errors/min"
+end
+
+Logster.register_rate_limit_per_hour([Logger::WARN, Logger::ERROR, Logger::FATAL], 60) do |rate|
+  puts "O no! The error rate is now #{rate} errors/hour"
+end
+```
+
 ### Note
 If you are seeing the error `No such middleware to insert before: ActionDispatch::DebugExceptions` after installing logster,
 then you are using a conflicting gem like `better_errors`.
@@ -59,7 +76,7 @@ Logster.store = Logster::RedisStore.new(redis_connection)
 ```
 
 ### Heroku Deployment
-In case you may be using the `rails_12factor` gem in a production deployment on Heroku, the standard `Rails.logger` will not cooperate properly with Logster. Extend Rails.logger in your `config/application.rb` or `config/initializers/logster.rb` with: 
+In case you may be using the `rails_12factor` gem in a production deployment on Heroku, the standard `Rails.logger` will not cooperate properly with Logster. Extend Rails.logger in your `config/application.rb` or `config/initializers/logster.rb` with:
 ```
 if Rails.env.production?
     Rails.logger.extend(ActiveSupport::Logger.broadcast(Logster.logger))
@@ -80,6 +97,9 @@ Logster UI is built using [Ember.js](http://emberjs.com/)
 
 # CHANGELOG
 
+- 2015-02-11: Version 1.1.1
+  - Feature: Error rate can now be tracked in one minute and one hour buckets.
+
 - 2015-11-27: Version 1.0.1
   - New assets and logster logo
   - Added favicon

data/lib/logster/base_store.rb

@@ -65,10 +65,22 @@ module Logster
       not_implemented
     end
 
+    # Registers a rate limit on the given severities of logs
+    def register_rate_limit(severities, limit, duration, &block)
+      not_implemented
+    end
+
+    # Checks all the existing rate limiters to check if any has been exceeded
+    def check_rate_limits(severity)
+      not_implemented
+    end
+
     def report(severity, progname, msg, opts = {})
       return if (!msg || (String === msg && msg.empty?)) && skip_empty
       return if level && severity < level
 
+      check_rate_limits(severity)
+
       message = Logster::Message.new(severity, progname, msg, opts[:timestamp])
 
       env = opts[:env] || {}

data/lib/logster/configuration.rb

@@ -1,6 +1,8 @@
 module Logster
   class Configuration
-    attr_accessor :current_context, :allow_grouping, :environments, :application_version, :web_title
+    attr_accessor :current_context, :allow_grouping, :environments,
+      :application_version, :web_title, :redis_prefix, :redis_raw_connection
+
     attr_writer :subdirectory
 
     def initialize

data/lib/logster/redis_store.rb

@@ -2,6 +2,88 @@ require 'json'
 require 'logster/base_store'
 
 module Logster
+  class RedisRateLimiter
+    BUCKETS = 6
+
+    attr_reader :key, :callback_key
+
+    def initialize(redis, severities, limit, duration, callback = nil)
+      @redis = use_raw_connection? ? Logster.config.redis_raw_connection : redis
+      @severities = severities
+      @limit = limit
+      @duration = duration
+      @callback = callback
+
+      # "_LOGSTER_RATE_LIMIT:012:20:30"
+      # Triggers callback when log levels of :debug, :info and :warn occurs 20 times within 30 secs
+      @key = "#{key_prefix}#{@severities.join("")}:#{@limit}:#{@duration}"
+      @callback_key = "#{@key}:callback_triggered"
+      @bucket_range = @duration / BUCKETS
+    end
+
+    def check(severity)
+      return unless @severities.include?(severity)
+      time = Time.now.to_i
+      num = bucket_number(time)
+      redis_key = "#{@key}:#{num}"
+
+      current_rate = @redis.eval <<-LUA
+        local bucket_number = #{num}
+        local bucket_count = redis.call("INCR", "#{redis_key}")
+
+        if bucket_count == 1 then
+          redis.call("EXPIRE", "#{redis_key}", "#{bucket_expiry(time)}")
+          redis.call("DEL", "#{callback_key}")
+        end
+
+        local function retrieve_rate ()
+          local sum = 0
+          local values = redis.call("MGET", #{mget_keys(num)})
+          for index, value in ipairs(values) do
+            if value ~= false then sum = sum + value end
+          end
+          return sum
+        end
+
+        return (retrieve_rate() + bucket_count)
+      LUA
+
+      if !@redis.get(@callback_key) && (current_rate >= @limit)
+        @callback.call(current_rate) if @callback
+        @redis.set(@callback_key, 1)
+      end
+
+      current_rate
+    end
+
+    private
+
+    def key_prefix
+      prefix = "__LOGSTER__RATE_LIMIT:".freeze
+      prefix = "#{Logster.config.redis_prefix}:#{prefix}" if use_raw_connection?
+      prefix
+    end
+
+    def mget_keys(bucket_num)
+      @mget_keys ||= (0..(BUCKETS - 1)).map { |i| "\"#{key}:#{i}\"" }
+      keys = @mget_keys.dup
+      keys.delete_at(bucket_num)
+      keys.join(", ")
+    end
+
+    def bucket_number(time)
+      (time % @duration) / @bucket_range
+    end
+
+    def bucket_expiry(time)
+      @duration - ((time % @duration) % @bucket_range)
+    end
+
+    def use_raw_connection?
+      Logster.config.redis_prefix && Logster.config.redis_raw_connection
+    end
+  end
+
   class RedisStore < BaseStore
 
     attr_accessor :redis, :max_backlog
@@ -179,8 +261,20 @@ module Logster
       @redis.hkeys(solved_key) || []
     end
 
+    def register_rate_limit_per_minute(severities, limit, &block)
+      register_rate_limit(severities, limit, 60, block)
+    end
+
+    def register_rate_limit_per_hour(severities, limit, &block)
+      register_rate_limit(severities, limit, 3600, block)
+    end
+
     protected
 
+    def rate_limits
+      @rate_limits ||= []
+    end
+
     def clear_solved(count = nil)
 
       ignores = Set.new(@redis.hkeys(solved_key) || [])
@@ -300,6 +394,10 @@ module Logster
 
     end
 
+    def check_rate_limits(severity)
+      rate_limits.each { |rate_limit| rate_limit.check(severity) }
+    end
+
     def solved_key
       @solved_key ||= "__LOGSTER__SOLVED_MAP"
     end
@@ -319,5 +417,14 @@ module Logster
     def grouping_key
       @grouping_key ||= "__LOGSTER__GMAP"
     end
+
+    private
+
+    def register_rate_limit(severities, limit, duration, callback)
+      severities = [severities] unless severities.is_a?(Array)
+      rate_limiter = RedisRateLimiter.new(@redis, severities, limit, duration, callback)
+      rate_limits << rate_limiter
+      rate_limiter
+    end
   end
 end

data/lib/logster/version.rb

@@ -1,3 +1,3 @@
 module Logster
-  VERSION = "1.0.1"
+  VERSION = "1.1.1"
 end

data/logster.gemspec

@@ -29,4 +29,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "redis"
   spec.add_development_dependency "guard"
   spec.add_development_dependency "guard-minitest"
+  spec.add_development_dependency "timecop"
 end

data/test/logster/test_redis_rate_limiter.rb

@@ -0,0 +1,178 @@
+require_relative '../test_helper'
+require 'logster/redis_store'
+require 'rack'
+
+class TestRedisRateLimiter < Minitest::Test
+  def setup
+    @redis = Redis.new
+  end
+
+  def teardown
+    @redis.flushall
+  end
+
+  def test_check
+    time = Time.new(2015, 1, 1, 1, 1)
+    Timecop.freeze(time)
+    called = 0
+
+    @rate_limiter = Logster::RedisRateLimiter.new(
+      @redis, [Logger::WARN], 8, 60, Proc.new { called += 1 }
+    )
+
+    assert_equal(1, @rate_limiter.check(Logger::WARN))
+    assert_redis_key(60, 0)
+    assert_equal(1, number_of_buckets)
+
+    Timecop.freeze(time + 10) do
+      assert_equal(2, @rate_limiter.check(Logger::WARN))
+      assert_redis_key(60, 1)
+      assert_equal(3, @rate_limiter.check(Logger::WARN))
+      assert_equal(2, number_of_buckets)
+    end
+
+    Timecop.freeze(time + 20) do
+      assert_equal(4, @rate_limiter.check(Logger::WARN))
+      assert_redis_key(60, 2)
+      assert_equal(3, number_of_buckets)
+    end
+
+    Timecop.freeze(time + 30) do
+      assert_equal(5, @rate_limiter.check(Logger::WARN))
+      assert_redis_key(60, 3)
+      assert_equal(4, number_of_buckets)
+    end
+
+    Timecop.freeze(time + 40) do
+      assert_equal(6, @rate_limiter.check(Logger::WARN))
+      assert_redis_key(60, 4)
+      assert_equal(5, number_of_buckets)
+    end
+
+    Timecop.freeze(time + 50) do
+      assert_equal(7, @rate_limiter.check(Logger::WARN))
+      assert_redis_key(60, 5)
+      assert_equal(6, number_of_buckets)
+    end
+
+    Timecop.freeze(time + 60) do
+      @redis.del("#{key}:0")
+      assert_equal(5, number_of_buckets)
+
+      assert_equal(7, @rate_limiter.check(Logger::WARN))
+      assert_redis_key(60, 0)
+      assert_equal(6, number_of_buckets)
+
+      assert_equal(8, @rate_limiter.check(Logger::WARN))
+      assert_equal(1, called)
+      assert_equal(6, number_of_buckets)
+      assert_equal("1", @redis.get(@rate_limiter.callback_key))
+    end
+
+    Timecop.freeze(time + 70) do
+      @redis.del("#{key}:1")
+      assert_equal(7, @rate_limiter.check(Logger::WARN))
+      assert_equal(nil, @redis.get(@rate_limiter.callback_key))
+    end
+  end
+
+  def test_check_with_multiple_severities
+    time = Time.new(2015, 1, 1, 1, 1)
+    Timecop.freeze(time)
+    called = 0
+
+    @rate_limiter = Logster::RedisRateLimiter.new(
+      @redis, [Logger::WARN, Logger::ERROR], 4, 60, Proc.new { called += 1 }
+    )
+
+    assert_equal(1, @rate_limiter.check(Logger::WARN))
+    assert_equal(2, @rate_limiter.check(Logger::ERROR))
+
+    Timecop.freeze(time + 50) do
+      assert_equal(3, @rate_limiter.check(Logger::WARN))
+      assert_equal(4, @rate_limiter.check(Logger::ERROR))
+      assert_equal(2, number_of_buckets)
+    end
+
+    assert_equal(5, @rate_limiter.check(Logger::ERROR))
+    assert_equal(1, called)
+  end
+
+  def test_bucket_number_per_minute
+    time = Time.new(2015, 1, 1, 1, 1)
+    Timecop.freeze(time)
+    @rate_limiter = Logster::RedisRateLimiter.new(@redis, [Logger::WARN], 1, 60)
+
+    assert_bucket_number(0, time)
+    assert_bucket_number(0, time + 9)
+    assert_bucket_number(1, time + 11)
+    assert_bucket_number(5, time + 59)
+  end
+
+  def test_bucket_number_per_hour
+    time = Time.new(2015, 1, 1, 1, 0)
+    Timecop.freeze(time)
+    @rate_limiter = Logster::RedisRateLimiter.new(@redis, [Logger::WARN], 1, 3600)
+
+    assert_bucket_number(0, time)
+    assert_bucket_number(1, time + 1199)
+    assert_bucket_number(2, time + 1200)
+    assert_bucket_number(5, time + 3599)
+  end
+
+  def test_bucket_expiry
+    time = Time.new(2015, 1, 1, 1, 1)
+    Timecop.freeze(time)
+    @rate_limiter = Logster::RedisRateLimiter.new(@redis, [Logger::WARN], 1, 60)
+
+    assert_bucket_expiry(60, time)
+    assert_bucket_expiry(55, time + 5)
+    assert_bucket_expiry(60, time + 10)
+    assert_bucket_expiry(58, time + 12)
+    assert_bucket_expiry(55, time + 15)
+    assert_bucket_expiry(51, time + 19)
+    assert_bucket_expiry(60, time + 20)
+    assert_bucket_expiry(55, time + 35)
+  end
+
+  def test_raw_connection
+    time = Time.new(2015, 1, 1, 1, 1)
+    Timecop.freeze(time)
+    Logster.config.redis_prefix = "lobster"
+    Logster.config.redis_raw_connection = @redis
+
+    @rate_limiter = Logster::RedisRateLimiter.new(nil, [Logger::WARN], 1, 60)
+
+    assert_equal(1, @rate_limiter.check(Logger::WARN))
+    assert_includes(key, "lobster")
+    assert_redis_key(60, 0)
+  end
+
+  private
+
+  def key
+    @rate_limiter.key
+  end
+
+  def number_of_buckets
+    @redis.keys("#{key}:[0-#{Logster::RedisRateLimiter::BUCKETS}]").size
+  end
+
+  def assert_bucket_number(expected, time)
+    Timecop.freeze(time) do
+      assert_equal(expected, @rate_limiter.send(:bucket_number, Time.now.to_i))
+    end
+  end
+
+  def assert_bucket_expiry(expected, time)
+    Timecop.freeze(time) do
+      assert_equal(expected, @rate_limiter.send(:bucket_expiry, Time.now.to_i))
+    end
+  end
+
+  def assert_redis_key(expected_ttl, expected_bucket_number)
+    redis_key = "#{key}:#{expected_bucket_number}"
+    assert(@redis.get(redis_key))
+    assert_equal(expected_ttl, @redis.ttl(redis_key))
+  end
+end

data/test/logster/test_redis_store.rb

@@ -326,4 +326,19 @@ class TestRedisStore < Minitest::Test
     assert_equal(orig, env)
   end
 
+  %w{minute hour}.each do |duration|
+    define_method "test_register_rate_limit_per_#{duration}" do
+      called = false
+
+      assert_instance_of(
+        Logster::RedisRateLimiter,
+        @store.public_send("register_rate_limit_per_#{duration}", Logger::WARN, 0) do
+          called = true
+        end
+      )
+
+      @store.report(Logger::WARN, "test", "test")
+      assert called
+    end
+  end
 end

data/test/test_helper.rb

@@ -5,6 +5,7 @@ require 'minitest/pride'
 require 'redis'
 require 'logster'
 require 'logster/base_store'
+require 'timecop'
 
 class Logster::TestStore < Logster::BaseStore
   attr_accessor :reported
@@ -28,5 +29,9 @@ class Logster::TestStore < Logster::BaseStore
     @reported = []
   end
 
+  def check_rate_limits(severity)
+    # Do nothing
+  end
+
   # get, protect, unprotect: unimplemented
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logster
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.1
 platform: ruby
 authors:
 - UI for viewing logs in Rack
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-27 00:00:00.000000000 Z
+date: 2016-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: UI for viewing logs in Rack
 email:
 - sam.saffron@gmail.com
@@ -102,6 +116,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -160,6 +175,7 @@ files:
 - test/logster/test_ignore_pattern.rb
 - test/logster/test_logger.rb
 - test/logster/test_message.rb
+- test/logster/test_redis_rate_limiter.rb
 - test/logster/test_redis_store.rb
 - test/test_helper.rb
 - vendor/assets/javascripts/logster.js.erb
@@ -183,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: UI for viewing logs in Rack
@@ -197,5 +213,6 @@ test_files:
 - test/logster/test_ignore_pattern.rb
 - test/logster/test_logger.rb
 - test/logster/test_message.rb
+- test/logster/test_redis_rate_limiter.rb
 - test/logster/test_redis_store.rb
 - test/test_helper.rb