rails_log_parser 0.0.2 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea15a1a3ec5a45569f3242ec132c7ad247d545c827cc8583439990095612099e
-  data.tar.gz: 4ad9958d1b198976eeb78cde4b8527d9402d90cdfe50ee3737de803de7a43cc6
+  metadata.gz: 2f608985d7c53f5ac09850df6c94d63437c5c1ecec18390072e83b4751a57e16
+  data.tar.gz: 68606ff5cea0948ad95cdd6f0ae583374b73654c013e95807dc0242070291879
 SHA512:
-  metadata.gz: ce42b4516aceb6284d2c51f551cadbb46b1ae53a2cc313a3cec003f423d095db6de3b269cb013b14797e9a5b5949f1051d25ae06a0c12f4a8fa4d9e63c75e3ef
-  data.tar.gz: edd3514b50cf68188ae011230d73c692695d6abaff9f3146436f58f1b0f39621221bc0ba760be337ad8e2d375438a71de7f367ee53f584564282dc2acbf0d0d5
+  metadata.gz: 7351dc20aa2fc84268825e955ef6b8ffe924571013c27e6b0728555442aa205c10600a5266c153387109fb9fe7f6bc03d2657fecc50956622a2ac7eb9e6339bc
+  data.tar.gz: eb0689bc7193fc130aae3cc40dd68ad5fc865add71696bd9a044442ef3094e61a26966f0044967cf8939365f0b03cf64d6b9b96ee5dc559236f0c80eb5770634

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.1)
+    rails_log_parser (0.0.6)
       enumerize (~> 2.4)
+      json
 
 GEM
   remote: https://rubygems.org/
@@ -19,6 +20,7 @@ GEM
       activesupport (>= 3.2)
     i18n (1.8.11)
       concurrent-ruby (~> 1.0)
+    json (2.3.1)
     minitest (5.14.4)
     rake (12.3.3)
     rspec (3.10.0)

data/README.md

@@ -30,7 +30,8 @@ Call the rake tasks in cronjobs:
 
 ```
 LOG_PATH=/srv/rails/log/production.log
-0,20,40 * * * * rake rails_log_parser:parse[22]' # summary of the last 22 minutes
+0,20,40 * * * * rake rails_log_parser:parse[22]'      # summary of the last 22 minutes
+59     23 * * * rake rails_log_parser:parse[22,true]' # summary of the last 22 minutes and save and analyse heuristic
 ```
 
 Or use it in your code:
@@ -40,8 +41,30 @@ parser = RailsLogParser::Parser.from_file(log_path)
 puts parser.actions.select(&:fatal?).map(&:headline)
 ```
 
+```ruby
+parser = RailsLogParser::Parser.from_file(log_path)
+parser.enable_heuristic(File.dirname(log_path)) # path to save heuristic stats
+print parser.summary(last_minutes: 22)          # print summary for the last 22 minutes
+```
+
 ## Changelog
 
+### 0.0.6
+
+* Adding heuristic to rate known exceptions
+
+### 0.0.5
+
+* Removing `URI::InvalidURIError` as known exceptions
+
+### 0.0.4
+
+* Handle stacktrace of fatals too
+
+### 0.0.3
+
+* Adding `URI::InvalidURIError` as known exceptions
+
 ### 0.0.2
 
 * Adding `ActionController::InvalidAuthenticityToken` as known exceptions

data/lib/rails_log_parser/action.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'time'
 require 'securerandom'
 
@@ -12,12 +13,13 @@ class RailsLogParser::Action
     'ActiveRecord::RecordNotFound' => :fatal,
     'ActionController::RoutingError' => :fatal,
     "Can't verify CSRF token authenticity." => :warn,
-    "ActionController::InvalidAuthenticityToken" => :fatal,
+    'ActionController::InvalidAuthenticityToken' => :fatal,
   }.freeze
 
   extend Enumerize
   enumerize :severity, in: SEVERITIES, predicates: true
   enumerize :type, in: %i[request without_request delayed_job active_job], predicates: true
+  attr_reader :datetime
 
   def initialize(type, id)
     self.type = type
@@ -35,9 +37,9 @@ class RailsLogParser::Action
     @headline = nil
   end
 
-  def known_exception?
+  def known_exception?(key = nil)
     @messages.any? do |message|
-      KNOWN_EXCEPTIONS.any? { |e, s| message.include?(e) && severity == s }
+      KNOWN_EXCEPTIONS.any? { |e, s| message.include?(e) && severity == s && (key.nil? || key == e) }
     end
   end
 

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'json'
+
+RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
+  attr_reader :stats
+
+  class << self
+    def build_heuristic(path, today)
+      sums = { actions: 0 }
+      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+        sums[exception.to_sym] = 0
+      end
+      7.times do |i|
+        stats = RailsLogParser::HeuristicStatFile.new(path, today.date - (i + 1)).load_stats
+        sums[:actions] += stats[:actions].to_i
+        RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+          sums[exception.to_sym] += stats.dig(:known_exceptions, exception.to_sym).to_i
+        end
+      end
+      output = {}
+      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+        next if sums[:actions] == 0
+
+        quota = sums[exception.to_sym].to_f / sums[:actions]
+        today_quota = today.rate(exception)
+        next if today_quota == 0
+
+        rate = ((today_quota - quota) / quota) / Math.sqrt(sums[:actions].to_f)
+        output[exception] = rate if rate > heuristic_threshold
+      end
+      output
+    end
+
+    def heuristic_threshold
+      @heuristic_threshold ||= ENV['RAILS_LOG_PARSER_THRESHOLD_HEURISTIC'] || RailsLogParser::THRESHOLD_HEURISTIC
+    end
+  end
+
+  def write_stats(actions)
+    actions = actions.select { |action| action.datetime.to_date == date }.sort_by(&:datetime)
+    @stats = {
+      actions: actions.count,
+      known_exceptions: {},
+      starts_at: actions.first&.datetime,
+      ends_at: actions.last&.datetime,
+    }
+
+    RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+      @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
+    end
+    File.write(heuristic_file_path, @stats.to_json)
+  end
+
+  def load_stats
+    @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
+    @stats ||= {}
+  rescue JSON::ParserError
+    @stats = {}
+  end
+
+  def heuristic_file_path
+    @heuristic_file_path ||= File.join(path, "heuristic_stats_#{date}.json")
+  end
+
+  def rate(exception)
+    return 0 if stats[:actions] == 0
+
+    stats.dig(:known_exceptions, exception.to_sym).to_f / stats[:actions]
+  end
+end

data/lib/rails_log_parser/line.rb

@@ -96,7 +96,7 @@ class RailsLogParser::Line
       return
     end
 
-    if parser.last_action&.error?
+    if parser.last_action&.error? || parser.last_action&.fatal?
       parser.last_action.add_stacktrace(line)
       return
     end

data/lib/rails_log_parser/parser.rb

@@ -24,6 +24,12 @@ class RailsLogParser::Parser
   def initialize
     @actions = {}
     @not_parseable_lines = []
+    @heuristic = nil
+  end
+
+  def enable_heuristic(path)
+    @heuristic = path
+    @heuristic_today = RailsLogParser::HeuristicStatFile.new(@heuristic, Date.today).tap { |p| p.write_stats(actions) }
   end
 
   def summary(last_minutes: nil)
@@ -32,23 +38,32 @@ class RailsLogParser::Parser
       from = last_minutes.to_i.minutes.ago
       relevant = relevant.select { |a| a.after?(from) }
     end
-    @summary_output = []
+    summary_output = []
     if @not_parseable_lines.present?
-      @summary_output.push('Not parseable lines:')
-      @summary_output += @not_parseable_lines.map { |line| "  #{line}" }
-      @summary_output.push("\n\n")
+      summary_output.push('Not parseable lines:')
+      summary_output += @not_parseable_lines.map { |line| "  #{line}" }
+      summary_output.push("\n\n")
     end
 
     %i[warn error fatal].each do |severity|
       selected = relevant.select { |a| a.public_send("#{severity}?") }.reject(&:known_exception?)
       next if selected.blank?
 
-      @summary_output.push("#{selected.count} lines with #{severity}:")
-      @summary_output += selected.map(&:headline).map { |line| "  #{line}" }
-      @summary_output.push("\n\n")
+      summary_output.push("#{selected.count} lines with #{severity}:")
+      summary_output += selected.map(&:headline).map { |line| "  #{line}" }
+      summary_output.push("\n\n")
+    end
+
+    unless @heuristic.nil?
+      stats = RailsLogParser::HeuristicStatFile.build_heuristic(@heuristic, @heuristic_today)
+      if stats.present?
+        summary_output.push('Heuristic match!')
+        stats.each { |k, v| summary_output.push("- #{k}: #{v}") }
+      end
+      summary_output.push("\n\n")
     end
 
-    @summary_output.join("\n")
+    summary_output.join("\n")
   end
 
   def actions

data/lib/rails_log_parser/tasks.rb

@@ -2,7 +2,9 @@
 
 namespace :rails_log_parser do
   desc 'notify about found problems in production.log'
-  task :parse, [:from_minutes] do |_t, args|
-    print RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path).summary(last_minutes: args[:from_minutes])
+  task :parse, [:from_minutes, :heuristic] do |_t, args|
+    parser = RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path)
+    parser.enable_heuristic(File.dirname(RailsLogParser::Parser.log_path)) if args[:heuristic] == 'true'
+    print parser.summary(last_minutes: args[:from_minutes])
   end
 end

data/lib/rails_log_parser.rb

@@ -3,10 +3,12 @@
 require 'enumerize'
 
 module RailsLogParser
+  THRESHOLD_HEURISTIC = 0.01
 end
 
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
+require_relative 'rails_log_parser/heuristic_stat_file'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.2'
+  spec.version       = '0.0.6'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 
@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'enumerize', '~> 2.4'
+  spec.add_dependency 'json'
   spec.add_development_dependency 'rspec'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.6
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-29 00:00:00.000000000 Z
+date: 2021-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -55,6 +69,7 @@ files:
 - Rakefile
 - lib/rails_log_parser.rb
 - lib/rails_log_parser/action.rb
+- lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
 - lib/rails_log_parser/parser.rb
 - lib/rails_log_parser/railtie.rb