rails_log_parser 0.0.7 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c223d9d32b464e7247934f4324ed4f4e38c64a9f6714737703b01578de4b3fe2
-  data.tar.gz: 540d452e4d79f74725ef369920f1fbb2a11cb8a5392316708779fe253d1ebb14
+  metadata.gz: eb02caffd4cab2119b027687a4fd22edd87f041b3dd718362bfdf7d7f6d5983c
+  data.tar.gz: b40124289928b7dd44bc5e91c654df4544734f6c696140b351ee7abdda1da4a3
 SHA512:
-  metadata.gz: 65053ab7d6218dbddff70035b00bdb719efea68f617a1d109a178e24e3cd86afbe734245a6326a6ee305330a81dd7831c059075e741953bbac07c116c3c8dc05
-  data.tar.gz: 47e107f64fa6ffbe6780268960884ffe19206de3cbfe74a2255fb6194e0ede28be2a00aa456c0edbbc7d0a52cc0f6f92284b54a7c48a793a5b75509a2b441daa
+  metadata.gz: 1bbb6a44f2f2f5f7ad4f53c5e73ede2103eed928ba0198d248b5d7d5ee82e13066f46143acb13023cc1408ca90c48f4e1acfd97b1d08efd0b8fa823809f890a5
+  data.tar.gz: 8b93bd4350b41c00b808c44ae8b61b68b45aa9af7bad2a2d8dddfa16b75f0d2efeffd49124fc216f2375f0ef1626d95b047e48c84138c2b5ebc88fb6e87c74d0

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.7)
+    rails_log_parser (0.0.10)
       enumerize (~> 2.4)
       json (>= 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (6.1.4.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -16,12 +16,12 @@ GEM
       zeitwerk (~> 2.3)
     concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
-    enumerize (2.4.0)
+    enumerize (2.5.0)
       activesupport (>= 3.2)
     i18n (1.8.11)
       concurrent-ruby (~> 1.0)
     json (2.6.1)
-    minitest (5.14.4)
+    minitest (5.15.0)
     rake (12.3.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -38,7 +38,7 @@ GEM
     rspec-support (3.10.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.3)
 
 PLATFORMS
   ruby

data/README.md

@@ -49,6 +49,20 @@ print parser.summary(last_minutes: 22)          # print summary for the last 22
 
 ## Changelog
 
+### 0.0.10
+
+* Ignore periods with too few requests
+
+### 0.0.9
+
+* Message not parseable lines only once a day
+* Delete old heuristic stat files automaticly
+
+### 0.0.8
+
+* Adding `ActionController::UnfilteredParameters` as known exceptions
+* Adjust heuristic rate for better matching
+
 ### 0.0.7
 
 * Remove empty lines on summary without report

data/lib/rails_log_parser/action.rb

@@ -14,6 +14,7 @@ class RailsLogParser::Action
     'ActionController::RoutingError' => :fatal,
     "Can't verify CSRF token authenticity." => :warn,
     'ActionController::InvalidAuthenticityToken' => :fatal,
+    'ActionController::UnfilteredParameters' => :fatal,
   }.freeze
 
   extend Enumerize

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -11,7 +11,7 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
       RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
         sums[exception.to_sym] = 0
       end
-      7.times do |i|
+      10.times do |i|
         stats = RailsLogParser::HeuristicStatFile.new(path, today.date - (i + 1)).load_stats
         sums[:actions] += stats[:actions].to_i
         RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
@@ -20,9 +20,10 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
       end
       output = {}
       RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-        next if sums[:actions] == 0
+        next if sums[:actions] < heuristic_min_actions
 
         quota = sums[exception.to_sym].to_f / sums[:actions]
+        next if quota == 0
         today_quota = today.rate(exception)
         next if today_quota == 0
 
@@ -35,6 +36,10 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
     def heuristic_threshold
       @heuristic_threshold ||= ENV['RAILS_LOG_PARSER_THRESHOLD_HEURISTIC'] || RailsLogParser::THRESHOLD_HEURISTIC
     end
+
+    def heuristic_min_actions
+      @heuristic_min_actions ||= ENV['RAILS_LOG_PARSER_MIN_ACTIONS_HEURISTIC'] || RailsLogParser::MIN_ACTIONS_HEURISTIC
+    end
   end
 
   def write_stats(actions)
@@ -49,9 +54,18 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
     RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
       @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
     end
+
+    delete_old_stats
     File.write(heuristic_file_path, @stats.to_json)
   end
 
+  def delete_old_stats
+    last_20_days = (0..19).map { |i| (Date.today - i) }.map { |date| File.join(path, "heuristic_stats_#{date}.json") }
+    Dir[File.join(path, 'heuristic_stats_*.json')].reject { |file| last_20_days.include?(file) }.each do |file|
+      File.unlink(file)
+    end
+  end
+
   def load_stats
     @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
     @stats ||= {}

data/lib/rails_log_parser/not_parseable_lines.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'json'
+
+class RailsLogParser::NotParseableLines
+  attr_reader :lines
+
+  def initialize
+    @lines = []
+    @path = File.join(File.dirname(RailsLogParser::Parser.log_path), 'not_parseable_lines.json')
+    load_file
+  end
+
+  def push(line)
+    @lines.push(line) unless today_lines.include?(line)
+  end
+
+  def save
+    @stats[Date.today.to_s] = today_lines + lines
+
+    last_7_days = (0..6).map { |i| (Date.today - i) }.map(&:to_s)
+    @stats.each_key do |key|
+      @stats.delete(key) unless last_7_days.include?(key)
+    end
+    File.write(@path, @stats.to_json)
+  end
+
+  protected
+
+  def today_lines
+    @stats[Date.today.to_s] || []
+  end
+
+  def load_file
+    @stats = JSON.parse(File.read(@path))
+    @stats ||= {}
+  rescue JSON::ParserError, Errno::ENOENT
+    @stats = {}
+  end
+end

data/lib/rails_log_parser/parser.rb

@@ -23,7 +23,7 @@ class RailsLogParser::Parser
 
   def initialize
     @actions = {}
-    @not_parseable_lines = []
+    @not_parseable_lines = RailsLogParser::NotParseableLines.new
     @heuristic = nil
   end
 
@@ -39,10 +39,11 @@ class RailsLogParser::Parser
       relevant = relevant.select { |a| a.after?(from) }
     end
     summary_output = []
-    if @not_parseable_lines.present?
+    if @not_parseable_lines.lines.present?
       summary_output.push('Not parseable lines:')
-      summary_output += @not_parseable_lines.map { |line| "  #{line}" }
+      summary_output += @not_parseable_lines.lines.map { |line| "  #{line}" }
       summary_output.push("\n\n")
+      @not_parseable_lines.save
     end
 
     %i[warn error fatal].each do |severity|
@@ -57,8 +58,8 @@ class RailsLogParser::Parser
     unless @heuristic.nil?
       stats = RailsLogParser::HeuristicStatFile.build_heuristic(@heuristic, @heuristic_today)
       if stats.present?
-        summary_output.push('Heuristic match!')
-        stats.each { |k, v| summary_output.push("- #{k}: #{v}") }
+        summary_output.push("Heuristic match! (threshold: #{RailsLogParser::HeuristicStatFile.heuristic_threshold})")
+        stats.each { |k, v| summary_output.push("- #{k}: #{v.round(4)}") }
         summary_output.push("\n\n")
       end
     end

data/lib/rails_log_parser.rb

@@ -3,12 +3,14 @@
 require 'enumerize'
 
 module RailsLogParser
-  THRESHOLD_HEURISTIC = 0.01
+  THRESHOLD_HEURISTIC = 0.02
+  MIN_ACTIONS_HEURISTIC = 10000
 end
 
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
 require_relative 'rails_log_parser/heuristic_stat_file'
+require_relative 'rails_log_parser/not_parseable_lines'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.7'
+  spec.version       = '0.0.10'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.10
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-15 00:00:00.000000000 Z
+date: 2022-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -71,6 +71,7 @@ files:
 - lib/rails_log_parser/action.rb
 - lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
+- lib/rails_log_parser/not_parseable_lines.rb
 - lib/rails_log_parser/parser.rb
 - lib/rails_log_parser/railtie.rb
 - lib/rails_log_parser/tasks.rb
@@ -96,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Simple and fast analysing of default rails logs