rails_log_parser 0.0.4 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 587c6ce2164707314caa439dbf465b895661df0e3d08de7ed3b10e0dcb9859a0
-  data.tar.gz: c4679547c0518a154aadf33ab33cc4cbac86abf15baec01767f975d4e7c48fd2
+  metadata.gz: c63fac78d3fda42cab291bcd30cc7abb739ed75ca82de98a72efb0587fd35954
+  data.tar.gz: d04eb23a0d4a37af92ce87a74ea900ef6b35ecb502e612ea559a5158afa76baf
 SHA512:
-  metadata.gz: 1f27b7ed6feb6677e0c70c05ac437c768823cfe7ba1daa967e26027e65b0810f3e391b55dc19bcf88a0590d5000c46d6178195f17946c05bff4fabed934be6a8
-  data.tar.gz: 69d6e3b2398d018236f6d5a5ec660011a9dc5b0c321b7ad5a0d8cec5a28c4a75f94306129ec2e909ff7a02ad0b36664b92aba9628c0c6fdb600cf3594c15f17b
+  metadata.gz: a5de5ab1e2853550540f58538743c1394332fd32895a061675cf4d39c0efda4a21c5d42a13045a9e78477350054403423b1b38264c2920e576aceb1658ac594f
+  data.tar.gz: 614f54e398d1f113cdd7345b05b51e402795dc1e2cc363ba9e1209f828cc3d928a008a6fd30ea7684382b2eb8affe59727f5d808fe8fb49df9bacd53a30e5ef2

data/Gemfile.lock

@@ -1,13 +1,14 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.3)
+    rails_log_parser (0.0.8)
       enumerize (~> 2.4)
+      json (>= 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (6.1.4.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -15,11 +16,12 @@ GEM
       zeitwerk (~> 2.3)
     concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
-    enumerize (2.4.0)
+    enumerize (2.5.0)
       activesupport (>= 3.2)
     i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
+    json (2.6.1)
+    minitest (5.15.0)
     rake (12.3.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -36,7 +38,7 @@ GEM
     rspec-support (3.10.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.3)
 
 PLATFORMS
   ruby

data/README.md

@@ -30,7 +30,8 @@ Call the rake tasks in cronjobs:
 
 ```
 LOG_PATH=/srv/rails/log/production.log
-0,20,40 * * * * rake rails_log_parser:parse[22]' # summary of the last 22 minutes
+0,20,40 * * * * rake rails_log_parser:parse[22]'      # summary of the last 22 minutes
+59     23 * * * rake rails_log_parser:parse[22,true]' # summary of the last 22 minutes and save and analyse heuristic
 ```
 
 Or use it in your code:
@@ -40,8 +41,31 @@ parser = RailsLogParser::Parser.from_file(log_path)
 puts parser.actions.select(&:fatal?).map(&:headline)
 ```
 
+```ruby
+parser = RailsLogParser::Parser.from_file(log_path)
+parser.enable_heuristic(File.dirname(log_path)) # path to save heuristic stats
+print parser.summary(last_minutes: 22)          # print summary for the last 22 minutes
+```
+
 ## Changelog
 
+### 0.0.8
+
+* Adding `ActionController::UnfilteredParameters` as known exceptions
+* Adjust heuristic rate for better matching
+
+### 0.0.7
+
+* Remove empty lines on summary without report
+
+### 0.0.6
+
+* Adding heuristic to rate known exceptions
+
+### 0.0.5
+
+* Removing `URI::InvalidURIError` as known exceptions
+
 ### 0.0.4
 
 * Handle stacktrace of fatals too

data/lib/rails_log_parser/action.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'time'
 require 'securerandom'
 
@@ -13,12 +14,13 @@ class RailsLogParser::Action
     'ActionController::RoutingError' => :fatal,
     "Can't verify CSRF token authenticity." => :warn,
     'ActionController::InvalidAuthenticityToken' => :fatal,
-    'URI::InvalidURIError' => :fatal,
+    'ActionController::UnfilteredParameters' => :fatal,
   }.freeze
 
   extend Enumerize
   enumerize :severity, in: SEVERITIES, predicates: true
   enumerize :type, in: %i[request without_request delayed_job active_job], predicates: true
+  attr_reader :datetime
 
   def initialize(type, id)
     self.type = type
@@ -36,9 +38,9 @@ class RailsLogParser::Action
     @headline = nil
   end
 
-  def known_exception?
+  def known_exception?(key = nil)
     @messages.any? do |message|
-      KNOWN_EXCEPTIONS.any? { |e, s| message.include?(e) && severity == s }
+      KNOWN_EXCEPTIONS.any? { |e, s| message.include?(e) && severity == s && (key.nil? || key == e) }
     end
   end
 

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'json'
+
+RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
+  attr_reader :stats
+
+  class << self
+    def build_heuristic(path, today)
+      sums = { actions: 0 }
+      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+        sums[exception.to_sym] = 0
+      end
+      10.times do |i|
+        stats = RailsLogParser::HeuristicStatFile.new(path, today.date - (i + 1)).load_stats
+        sums[:actions] += stats[:actions].to_i
+        RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+          sums[exception.to_sym] += stats.dig(:known_exceptions, exception.to_sym).to_i
+        end
+      end
+      output = {}
+      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+        next if sums[:actions] == 0
+
+        quota = sums[exception.to_sym].to_f / sums[:actions]
+        today_quota = today.rate(exception)
+        next if today_quota == 0
+
+        rate = ((today_quota - quota) / quota) / Math.sqrt(sums[:actions].to_f)
+        output[exception] = rate if rate > heuristic_threshold
+      end
+      output
+    end
+
+    def heuristic_threshold
+      @heuristic_threshold ||= ENV['RAILS_LOG_PARSER_THRESHOLD_HEURISTIC'] || RailsLogParser::THRESHOLD_HEURISTIC
+    end
+  end
+
+  def write_stats(actions)
+    actions = actions.select { |action| action.datetime.to_date == date }.sort_by(&:datetime)
+    @stats = {
+      actions: actions.count,
+      known_exceptions: {},
+      starts_at: actions.first&.datetime,
+      ends_at: actions.last&.datetime,
+    }
+
+    RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+      @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
+    end
+    File.write(heuristic_file_path, @stats.to_json)
+  end
+
+  def load_stats
+    @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
+    @stats ||= {}
+  rescue JSON::ParserError
+    @stats = {}
+  end
+
+  def heuristic_file_path
+    @heuristic_file_path ||= File.join(path, "heuristic_stats_#{date}.json")
+  end
+
+  def rate(exception)
+    return 0 if stats[:actions] == 0
+
+    stats.dig(:known_exceptions, exception.to_sym).to_f / stats[:actions]
+  end
+end

data/lib/rails_log_parser/parser.rb

@@ -24,6 +24,12 @@ class RailsLogParser::Parser
   def initialize
     @actions = {}
     @not_parseable_lines = []
+    @heuristic = nil
+  end
+
+  def enable_heuristic(path)
+    @heuristic = path
+    @heuristic_today = RailsLogParser::HeuristicStatFile.new(@heuristic, Date.today).tap { |p| p.write_stats(actions) }
   end
 
   def summary(last_minutes: nil)
@@ -32,23 +38,32 @@ class RailsLogParser::Parser
       from = last_minutes.to_i.minutes.ago
       relevant = relevant.select { |a| a.after?(from) }
     end
-    @summary_output = []
+    summary_output = []
     if @not_parseable_lines.present?
-      @summary_output.push('Not parseable lines:')
-      @summary_output += @not_parseable_lines.map { |line| "  #{line}" }
-      @summary_output.push("\n\n")
+      summary_output.push('Not parseable lines:')
+      summary_output += @not_parseable_lines.map { |line| "  #{line}" }
+      summary_output.push("\n\n")
     end
 
     %i[warn error fatal].each do |severity|
       selected = relevant.select { |a| a.public_send("#{severity}?") }.reject(&:known_exception?)
       next if selected.blank?
 
-      @summary_output.push("#{selected.count} lines with #{severity}:")
-      @summary_output += selected.map(&:headline).map { |line| "  #{line}" }
-      @summary_output.push("\n\n")
+      summary_output.push("#{selected.count} lines with #{severity}:")
+      summary_output += selected.map(&:headline).map { |line| "  #{line}" }
+      summary_output.push("\n\n")
+    end
+
+    unless @heuristic.nil?
+      stats = RailsLogParser::HeuristicStatFile.build_heuristic(@heuristic, @heuristic_today)
+      if stats.present?
+        summary_output.push("Heuristic match! (threshold: #{RailsLogParser::HeuristicStatFile.heuristic_threshold})")
+        stats.each { |k, v| summary_output.push("- #{k}: #{v.round(4)}") }
+        summary_output.push("\n\n")
+      end
     end
 
-    @summary_output.join("\n")
+    summary_output.join("\n")
   end
 
   def actions

data/lib/rails_log_parser/tasks.rb

@@ -2,7 +2,9 @@
 
 namespace :rails_log_parser do
   desc 'notify about found problems in production.log'
-  task :parse, [:from_minutes] do |_t, args|
-    print RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path).summary(last_minutes: args[:from_minutes])
+  task :parse, [:from_minutes, :heuristic] do |_t, args|
+    parser = RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path)
+    parser.enable_heuristic(File.dirname(RailsLogParser::Parser.log_path)) if args[:heuristic] == 'true'
+    print parser.summary(last_minutes: args[:from_minutes])
   end
 end

data/lib/rails_log_parser.rb

@@ -3,10 +3,12 @@
 require 'enumerize'
 
 module RailsLogParser
+  THRESHOLD_HEURISTIC = 0.02
 end
 
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
+require_relative 'rails_log_parser/heuristic_stat_file'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.4'
+  spec.version       = '0.0.8'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 
@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'enumerize', '~> 2.4'
-  spec.add_development_dependency 'rspec'
+  spec.add_dependency 'json', '>= 2.0'
+  spec.add_development_dependency 'rspec', '>= 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.8
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-30 00:00:00.000000000 Z
+date: 2022-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -24,20 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 description: If you want to get an email with errors and fatal log lines you can use
   this gem.
 email:
@@ -55,6 +69,7 @@ files:
 - Rakefile
 - lib/rails_log_parser.rb
 - lib/rails_log_parser/action.rb
+- lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
 - lib/rails_log_parser/parser.rb
 - lib/rails_log_parser/railtie.rb