rails_log_parser 0.0.15 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7075d8f72e2f41875ca8e7d7b96cc9895a09cf3561b1f7a8ce140decd8172f0c
-  data.tar.gz: 0c497f4bcb5a4802d0a2933f5af11b95498a60fadb57619fe13719441e8f49d8
+  metadata.gz: 96ae5461bc6323f83c7ed166c728e1c0d161b9a0a3fe921d723b7f13bac11db9
+  data.tar.gz: 6b9d0276cdef26834994ea8001d1942fd025d14ed6c59f2bab25cbbcfcd13d22
 SHA512:
-  metadata.gz: 1e9ec59884a21ac46292369dea9d3ff94426bff3d67cb4962cf220f5412448b8d366515837b12ea319cdfa2c7f34320819c9bdaf23ee4584bf3ebfb169478807
-  data.tar.gz: 1ede0e6934f9e56d2fe0e88b93835d08da872395fb5678ba4eabdd729ba4e167bbea90dda06aa6565e485f1e2ec48082607d30d37df11c7245a78c979d8f21f5
+  metadata.gz: 92d7e4cc9808c00d9d3a2ad13c0885c1b540d589cccf4188f77d9ada5b042d092f125898eb32d6dd13c57ff460c2f391b1cd7bbb9661d99b2e17ad58db7261f0
+  data.tar.gz: '08257c0b4050e94b2ccff09d55dc391444da9c92c1d090e9e6459b23a683f686c7bf2d942f7a7b62987f67b2767e3501ab1c36abddbadbb54cb29c0a35670417'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.15)
+    rails_log_parser (0.0.16)
       enumerize (~> 2.4)
       json (>= 2.0)
       rails (>= 2.1)
@@ -71,11 +71,13 @@ GEM
     marcel (1.0.1)
     method_source (1.0.0)
     mini_mime (1.1.0)
-    mini_portile2 (2.4.0)
+    mini_portile2 (2.8.5)
     minitest (5.18.0)
     nio4r (2.5.7)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
+    nokogiri (1.16.0)
+      mini_portile2 (~> 2.8.2)
+      racc (~> 1.4)
+    racc (1.7.3)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)

data/README.md

@@ -31,7 +31,6 @@ Call the rake tasks in cronjobs:
 ```
 LOG_PATH=/srv/rails/log/production.log
 0,20,40 * * * * rake rails_log_parser:parse[22]'      # summary of the last 22 minutes
-59     23 * * * rake rails_log_parser:parse[22,true]' # summary of the last 22 minutes and save and analyse heuristic
 ```
 
 Or use it in your code:
@@ -43,7 +42,6 @@ puts parser.actions.select(&:fatal?).map(&:headline)
 
 ```ruby
 parser = RailsLogParser::Parser.from_file(log_path)
-parser.enable_heuristic(File.dirname(log_path)) # path to save heuristic stats
 print parser.summary(last_minutes: 22)          # print summary for the last 22 minutes
 ```
 
@@ -62,6 +60,16 @@ end
 
 ## Changelog
 
+### 0.0.17
+
+* Adding `ActionDispatch::Http::MimeNegotiation::InvalidType` as known exceptions
+* Update dependencies
+
+### 0.0.16
+
+* Remove heuristic
+* Fix last action handling
+
 ### 0.0.15
 
 * Filter lines by config file

data/lib/rails_log_parser/action.rb

@@ -4,10 +4,6 @@ require 'time'
 require 'securerandom'
 
 class RailsLogParser::Action
-  class << self
-    attr_accessor :last
-  end
-
   SEVERITIES = %i[debug info warn error fatal].freeze
   KNOWN_EXCEPTIONS = {
     "Can't verify CSRF token authenticity." => :warn,
@@ -16,6 +12,7 @@ class RailsLogParser::Action
     'ActionController::UnfilteredParameters' => :fatal,
     'ActionController::UnknownFormat' => :fatal,
     'ActiveRecord::RecordNotFound' => :fatal,
+    'ActionDispatch::Http::MimeNegotiation::InvalidType' => :fatal,
   }.freeze
 
   extend Enumerize
@@ -28,7 +25,6 @@ class RailsLogParser::Action
     @id = id
     @messages = []
     @stacktrace = []
-    self.class.last = self
   end
 
   def severity=(value)

data/lib/rails_log_parser/parser.rb

@@ -5,7 +5,7 @@ class RailsLogParser::Parser
     attr_writer :log_path
 
     def log_path
-      @log_path || ENV['LOG_PATH']
+      @log_path || ENV['LOG_PATH'] || raise('no log_path given')
     end
 
     def from_file(path)
@@ -19,7 +19,7 @@ class RailsLogParser::Parser
     end
   end
 
-  attr_reader :not_parseable_lines
+  attr_reader :not_parseable_lines, :last_action
 
   def initialize
     config_file = File.join(Dir.pwd,'config/rails_log_parser.rb')
@@ -27,13 +27,8 @@ class RailsLogParser::Parser
 
     @actions = {}
     @not_parseable_lines = RailsLogParser::NotParseableLines.new
-    @heuristic = nil
   end
 
-  def enable_heuristic(path)
-    @heuristic = path
-    @heuristic_today = RailsLogParser::HeuristicStatFile.new(@heuristic, Date.today).tap { |p| p.write_stats(actions) }
-  end
 
   def summary(last_minutes: nil)
     relevant = actions
@@ -58,15 +53,6 @@ class RailsLogParser::Parser
       summary_output.push("\n\n")
     end
 
-    unless @heuristic.nil?
-      stats = RailsLogParser::HeuristicStatFile.build_heuristic(@heuristic, @heuristic_today)
-      if stats.present?
-        summary_output.push("Heuristic match! (threshold: #{RailsLogParser::HeuristicStatFile.heuristic_threshold})")
-        stats.each { |k, v| summary_output.push("- #{k}: #{v.round(4)}") }
-        summary_output.push("\n\n")
-      end
-    end
-
     summary_output.join("\n")
   end
 
@@ -83,6 +69,7 @@ class RailsLogParser::Parser
     @actions[params['id']].severity = params['severity_label']
     @actions[params['id']].datetime = params['datetime']
     @actions[params['id']].add_message(params['message']) unless params['message'].nil?
+    @last_action = @actions[params['id']]
   end
 
   def request(params)
@@ -113,8 +100,4 @@ class RailsLogParser::Parser
     @actions[params['id']] ||= RailsLogParser::Action.new(type, params['id'])
     @actions[params['id']].add_message(params['message'])
   end
-
-  def last_action
-    RailsLogParser::Action.last
-  end
 end

data/lib/rails_log_parser/tasks.rb

@@ -2,9 +2,8 @@
 
 namespace :rails_log_parser do
   desc 'notify about found problems in production.log'
-  task :parse, [:from_minutes, :heuristic] do |_t, args|
+  task :parse, [:from_minutes] do |_t, args|
     parser = RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path)
-    parser.enable_heuristic(File.dirname(RailsLogParser::Parser.log_path)) if args[:heuristic] == 'true'
     print parser.summary(last_minutes: args[:from_minutes])
   end
 end

data/lib/rails_log_parser.rb

@@ -4,9 +4,6 @@ require 'enumerize'
 require 'active_support/core_ext/module/attribute_accessors'
 
 module RailsLogParser
-  THRESHOLD_HEURISTIC = 0.02
-  MIN_ACTIONS_HEURISTIC = 100000 # sum of last 10 days
-
   mattr_accessor :ignore_lines, default: []
 
   def self.configure
@@ -17,7 +14,6 @@ end
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
-require_relative 'rails_log_parser/heuristic_stat_file'
 require_relative 'rails_log_parser/not_parseable_lines'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.15'
+  spec.version       = '0.0.17'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.17
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-05-24 00:00:00.000000000 Z
+date: 2024-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -83,7 +83,6 @@ files:
 - Rakefile
 - lib/rails_log_parser.rb
 - lib/rails_log_parser/action.rb
-- lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
 - lib/rails_log_parser/not_parseable_lines.rb
 - lib/rails_log_parser/parser.rb
@@ -111,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Simple and fast analysing of default rails logs

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
-  attr_reader :stats
-
-  class << self
-    def build_heuristic(path, today)
-      sums = { actions: 0 }
-      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-        sums[exception.to_sym] = 0
-      end
-      10.times do |i|
-        stats = RailsLogParser::HeuristicStatFile.new(path, today.date - (i + 1)).load_stats
-        sums[:actions] += stats[:actions].to_i
-        RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-          sums[exception.to_sym] += stats.dig(:known_exceptions, exception.to_sym).to_i
-        end
-      end
-      output = {}
-      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-        next if sums[:actions] < heuristic_min_actions
-
-        quota = sums[exception.to_sym].to_f / sums[:actions]
-        next if quota == 0
-        today_quota = today.rate(exception)
-        next if today_quota == 0
-
-        rate = ((today_quota - quota) / quota) / Math.sqrt(sums[:actions].to_f)
-        output[exception] = rate if rate > heuristic_threshold
-      end
-      output
-    end
-
-    def heuristic_threshold
-      @heuristic_threshold ||= ENV['RAILS_LOG_PARSER_THRESHOLD_HEURISTIC'] || RailsLogParser::THRESHOLD_HEURISTIC
-    end
-
-    def heuristic_min_actions
-      @heuristic_min_actions ||= ENV['RAILS_LOG_PARSER_MIN_ACTIONS_HEURISTIC'] || RailsLogParser::MIN_ACTIONS_HEURISTIC
-    end
-  end
-
-  def write_stats(actions)
-    actions = actions.select { |action| action.datetime.to_date == date }.sort_by(&:datetime)
-    @stats = {
-      actions: actions.count,
-      known_exceptions: {},
-      starts_at: actions.first&.datetime,
-      ends_at: actions.last&.datetime,
-    }
-
-    RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-      @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
-    end
-
-    delete_old_stats
-    File.write(heuristic_file_path, @stats.to_json)
-  end
-
-  def delete_old_stats
-    last_20_days = (0..19).map { |i| (Date.today - i) }.map { |date| File.join(path, "heuristic_stats_#{date}.json") }
-    Dir[File.join(path, 'heuristic_stats_*.json')].reject { |file| last_20_days.include?(file) }.each do |file|
-      File.unlink(file)
-    end
-  end
-
-  def load_stats
-    @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
-    @stats ||= {}
-  rescue JSON::ParserError
-    @stats = {}
-  end
-
-  def heuristic_file_path
-    @heuristic_file_path ||= File.join(path, "heuristic_stats_#{date}.json")
-  end
-
-  def rate(exception)
-    return 0 if stats[:actions] == 0
-
-    stats.dig(:known_exceptions, exception.to_sym).to_f / stats[:actions]
-  end
-end