rails_log_parser 0.0.3 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14be05b30152b307fd2bfc5aa398e2381fc3849c130ad21313aa74c1d5e533f2
-  data.tar.gz: b8ab4f231d38089960ce3dd8d04451fcf04ef53d94c381abc11eb6c35f874b97
+  metadata.gz: c223d9d32b464e7247934f4324ed4f4e38c64a9f6714737703b01578de4b3fe2
+  data.tar.gz: 540d452e4d79f74725ef369920f1fbb2a11cb8a5392316708779fe253d1ebb14
 SHA512:
-  metadata.gz: f49c2178c57daa4c470421d617d78460c5ff6fdebf28dc8141a9830655e1a6b6c3984690dce8ab602e70caf432b72e825e713846a46ba74ede5cd3eba1baf447
-  data.tar.gz: 21f1c5026a46d19fc6f6577dc3927788108df79b430a1ce6c3549c494e4845b184a4ce1a3ad730e16faf1f95e16e35d73374d91cd546fb9e983322b1d98d649c
+  metadata.gz: 65053ab7d6218dbddff70035b00bdb719efea68f617a1d109a178e24e3cd86afbe734245a6326a6ee305330a81dd7831c059075e741953bbac07c116c3c8dc05
+  data.tar.gz: 47e107f64fa6ffbe6780268960884ffe19206de3cbfe74a2255fb6194e0ede28be2a00aa456c0edbbc7d0a52cc0f6f92284b54a7c48a793a5b75509a2b441daa

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.3)
+    rails_log_parser (0.0.7)
       enumerize (~> 2.4)
+      json (>= 2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -19,6 +20,7 @@ GEM
       activesupport (>= 3.2)
     i18n (1.8.11)
       concurrent-ruby (~> 1.0)
+    json (2.6.1)
     minitest (5.14.4)
     rake (12.3.3)
     rspec (3.10.0)

data/README.md

@@ -30,7 +30,8 @@ Call the rake tasks in cronjobs:
 
 ```
 LOG_PATH=/srv/rails/log/production.log
-0,20,40 * * * * rake rails_log_parser:parse[22]' # summary of the last 22 minutes
+0,20,40 * * * * rake rails_log_parser:parse[22]'      # summary of the last 22 minutes
+59     23 * * * rake rails_log_parser:parse[22,true]' # summary of the last 22 minutes and save and analyse heuristic
 ```
 
 Or use it in your code:
@@ -40,8 +41,30 @@ parser = RailsLogParser::Parser.from_file(log_path)
 puts parser.actions.select(&:fatal?).map(&:headline)
 ```
 
+```ruby
+parser = RailsLogParser::Parser.from_file(log_path)
+parser.enable_heuristic(File.dirname(log_path)) # path to save heuristic stats
+print parser.summary(last_minutes: 22)          # print summary for the last 22 minutes
+```
+
 ## Changelog
 
+### 0.0.7
+
+* Remove empty lines on summary without report
+
+### 0.0.6
+
+* Adding heuristic to rate known exceptions
+
+### 0.0.5
+
+* Removing `URI::InvalidURIError` as known exceptions
+
+### 0.0.4
+
+* Handle stacktrace of fatals too
+
 ### 0.0.3
 
 * Adding `URI::InvalidURIError` as known exceptions

data/lib/rails_log_parser/action.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'time'
 require 'securerandom'
 
@@ -13,12 +14,12 @@ class RailsLogParser::Action
     'ActionController::RoutingError' => :fatal,
     "Can't verify CSRF token authenticity." => :warn,
     'ActionController::InvalidAuthenticityToken' => :fatal,
-    'URI::InvalidURIError' => :fatal,
   }.freeze
 
   extend Enumerize
   enumerize :severity, in: SEVERITIES, predicates: true
   enumerize :type, in: %i[request without_request delayed_job active_job], predicates: true
+  attr_reader :datetime
 
   def initialize(type, id)
     self.type = type
@@ -36,9 +37,9 @@ class RailsLogParser::Action
     @headline = nil
   end
 
-  def known_exception?
+  def known_exception?(key = nil)
     @messages.any? do |message|
-      KNOWN_EXCEPTIONS.any? { |e, s| message.include?(e) && severity == s }
+      KNOWN_EXCEPTIONS.any? { |e, s| message.include?(e) && severity == s && (key.nil? || key == e) }
     end
   end
 

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'json'
+
+RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
+  attr_reader :stats
+
+  class << self
+    def build_heuristic(path, today)
+      sums = { actions: 0 }
+      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+        sums[exception.to_sym] = 0
+      end
+      7.times do |i|
+        stats = RailsLogParser::HeuristicStatFile.new(path, today.date - (i + 1)).load_stats
+        sums[:actions] += stats[:actions].to_i
+        RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+          sums[exception.to_sym] += stats.dig(:known_exceptions, exception.to_sym).to_i
+        end
+      end
+      output = {}
+      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+        next if sums[:actions] == 0
+
+        quota = sums[exception.to_sym].to_f / sums[:actions]
+        today_quota = today.rate(exception)
+        next if today_quota == 0
+
+        rate = ((today_quota - quota) / quota) / Math.sqrt(sums[:actions].to_f)
+        output[exception] = rate if rate > heuristic_threshold
+      end
+      output
+    end
+
+    def heuristic_threshold
+      @heuristic_threshold ||= ENV['RAILS_LOG_PARSER_THRESHOLD_HEURISTIC'] || RailsLogParser::THRESHOLD_HEURISTIC
+    end
+  end
+
+  def write_stats(actions)
+    actions = actions.select { |action| action.datetime.to_date == date }.sort_by(&:datetime)
+    @stats = {
+      actions: actions.count,
+      known_exceptions: {},
+      starts_at: actions.first&.datetime,
+      ends_at: actions.last&.datetime,
+    }
+
+    RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
+      @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
+    end
+    File.write(heuristic_file_path, @stats.to_json)
+  end
+
+  def load_stats
+    @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
+    @stats ||= {}
+  rescue JSON::ParserError
+    @stats = {}
+  end
+
+  def heuristic_file_path
+    @heuristic_file_path ||= File.join(path, "heuristic_stats_#{date}.json")
+  end
+
+  def rate(exception)
+    return 0 if stats[:actions] == 0
+
+    stats.dig(:known_exceptions, exception.to_sym).to_f / stats[:actions]
+  end
+end

data/lib/rails_log_parser/line.rb

@@ -96,7 +96,7 @@ class RailsLogParser::Line
       return
     end
 
-    if parser.last_action&.error?
+    if parser.last_action&.error? || parser.last_action&.fatal?
       parser.last_action.add_stacktrace(line)
       return
     end

data/lib/rails_log_parser/parser.rb

@@ -24,6 +24,12 @@ class RailsLogParser::Parser
   def initialize
     @actions = {}
     @not_parseable_lines = []
+    @heuristic = nil
+  end
+
+  def enable_heuristic(path)
+    @heuristic = path
+    @heuristic_today = RailsLogParser::HeuristicStatFile.new(@heuristic, Date.today).tap { |p| p.write_stats(actions) }
   end
 
   def summary(last_minutes: nil)
@@ -32,23 +38,32 @@ class RailsLogParser::Parser
       from = last_minutes.to_i.minutes.ago
       relevant = relevant.select { |a| a.after?(from) }
     end
-    @summary_output = []
+    summary_output = []
     if @not_parseable_lines.present?
-      @summary_output.push('Not parseable lines:')
-      @summary_output += @not_parseable_lines.map { |line| "  #{line}" }
-      @summary_output.push("\n\n")
+      summary_output.push('Not parseable lines:')
+      summary_output += @not_parseable_lines.map { |line| "  #{line}" }
+      summary_output.push("\n\n")
     end
 
     %i[warn error fatal].each do |severity|
       selected = relevant.select { |a| a.public_send("#{severity}?") }.reject(&:known_exception?)
       next if selected.blank?
 
-      @summary_output.push("#{selected.count} lines with #{severity}:")
-      @summary_output += selected.map(&:headline).map { |line| "  #{line}" }
-      @summary_output.push("\n\n")
+      summary_output.push("#{selected.count} lines with #{severity}:")
+      summary_output += selected.map(&:headline).map { |line| "  #{line}" }
+      summary_output.push("\n\n")
+    end
+
+    unless @heuristic.nil?
+      stats = RailsLogParser::HeuristicStatFile.build_heuristic(@heuristic, @heuristic_today)
+      if stats.present?
+        summary_output.push('Heuristic match!')
+        stats.each { |k, v| summary_output.push("- #{k}: #{v}") }
+        summary_output.push("\n\n")
+      end
     end
 
-    @summary_output.join("\n")
+    summary_output.join("\n")
   end
 
   def actions

data/lib/rails_log_parser/tasks.rb

@@ -2,7 +2,9 @@
 
 namespace :rails_log_parser do
   desc 'notify about found problems in production.log'
-  task :parse, [:from_minutes] do |_t, args|
-    print RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path).summary(last_minutes: args[:from_minutes])
+  task :parse, [:from_minutes, :heuristic] do |_t, args|
+    parser = RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path)
+    parser.enable_heuristic(File.dirname(RailsLogParser::Parser.log_path)) if args[:heuristic] == 'true'
+    print parser.summary(last_minutes: args[:from_minutes])
   end
 end

data/lib/rails_log_parser.rb

@@ -3,10 +3,12 @@
 require 'enumerize'
 
 module RailsLogParser
+  THRESHOLD_HEURISTIC = 0.01
 end
 
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
+require_relative 'rails_log_parser/heuristic_stat_file'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.3'
+  spec.version       = '0.0.7'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 
@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'enumerize', '~> 2.4'
-  spec.add_development_dependency 'rspec'
+  spec.add_dependency 'json', '>= 2.0'
+  spec.add_development_dependency 'rspec', '>= 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.7
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-29 00:00:00.000000000 Z
+date: 2021-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -24,20 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 description: If you want to get an email with errors and fatal log lines you can use
   this gem.
 email:
@@ -55,6 +69,7 @@ files:
 - Rakefile
 - lib/rails_log_parser.rb
 - lib/rails_log_parser/action.rb
+- lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
 - lib/rails_log_parser/parser.rb
 - lib/rails_log_parser/railtie.rb
@@ -81,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Simple and fast analysing of default rails logs