rails_log_parser 0.0.8 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c63fac78d3fda42cab291bcd30cc7abb739ed75ca82de98a72efb0587fd35954
-  data.tar.gz: d04eb23a0d4a37af92ce87a74ea900ef6b35ecb502e612ea559a5158afa76baf
+  metadata.gz: 576171823f3fa4aacd3fe442e88b4fe02ac451bd2d43201ffde3e62ce1a7498e
+  data.tar.gz: 8cea97be36724fe58aa5d2d2ca740529c05eed3e1ce46bc33a104508cd31a6bb
 SHA512:
-  metadata.gz: a5de5ab1e2853550540f58538743c1394332fd32895a061675cf4d39c0efda4a21c5d42a13045a9e78477350054403423b1b38264c2920e576aceb1658ac594f
-  data.tar.gz: 614f54e398d1f113cdd7345b05b51e402795dc1e2cc363ba9e1209f828cc3d928a008a6fd30ea7684382b2eb8affe59727f5d808fe8fb49df9bacd53a30e5ef2
+  metadata.gz: 5feb9c35f0e05c35b91ab8f72dd9b0674a7e491154c05209c1bbc88a22e703a62cda6bb881c446a20c47fc2722de9f105d7e2bd03f8d49e6a95261909bf26b9c
+  data.tar.gz: e0ec7cffe45320c215f1581c82481133fd6ed979e6c872a32446bfdfd12f7c7b0f7c85f3b7846a9217aaf25bf15c92891c1f1290d8ec1d90c54bb19481e27596

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.8)
+    rails_log_parser (0.0.11)
       enumerize (~> 2.4)
       json (>= 2.0)
 
@@ -14,11 +14,11 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     diff-lcs (1.4.4)
     enumerize (2.5.0)
       activesupport (>= 3.2)
-    i18n (1.8.11)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     json (2.6.1)
     minitest (5.15.0)

data/README.md

@@ -49,6 +49,19 @@ print parser.summary(last_minutes: 22)          # print summary for the last 22
 
 ## Changelog
 
+### 0.0.11
+
+* Increase minimum requests value for heuristic
+
+### 0.0.10
+
+* Ignore periods with too few requests
+
+### 0.0.9
+
+* Message not parseable lines only once a day
+* Delete old heuristic stat files automaticly
+
 ### 0.0.8
 
 * Adding `ActionController::UnfilteredParameters` as known exceptions

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -20,9 +20,10 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
       end
       output = {}
       RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-        next if sums[:actions] == 0
+        next if sums[:actions] < heuristic_min_actions
 
         quota = sums[exception.to_sym].to_f / sums[:actions]
+        next if quota == 0
         today_quota = today.rate(exception)
         next if today_quota == 0
 
@@ -35,6 +36,10 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
     def heuristic_threshold
       @heuristic_threshold ||= ENV['RAILS_LOG_PARSER_THRESHOLD_HEURISTIC'] || RailsLogParser::THRESHOLD_HEURISTIC
     end
+
+    def heuristic_min_actions
+      @heuristic_min_actions ||= ENV['RAILS_LOG_PARSER_MIN_ACTIONS_HEURISTIC'] || RailsLogParser::MIN_ACTIONS_HEURISTIC
+    end
   end
 
   def write_stats(actions)
@@ -49,9 +54,18 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
     RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
       @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
     end
+
+    delete_old_stats
     File.write(heuristic_file_path, @stats.to_json)
   end
 
+  def delete_old_stats
+    last_20_days = (0..19).map { |i| (Date.today - i) }.map { |date| File.join(path, "heuristic_stats_#{date}.json") }
+    Dir[File.join(path, 'heuristic_stats_*.json')].reject { |file| last_20_days.include?(file) }.each do |file|
+      File.unlink(file)
+    end
+  end
+
   def load_stats
     @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
     @stats ||= {}

data/lib/rails_log_parser/not_parseable_lines.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'json'
+
+class RailsLogParser::NotParseableLines
+  attr_reader :lines
+
+  def initialize
+    @lines = []
+    @path = File.join(File.dirname(RailsLogParser::Parser.log_path), 'not_parseable_lines.json')
+    load_file
+  end
+
+  def push(line)
+    @lines.push(line) unless today_lines.include?(line)
+  end
+
+  def save
+    @stats[Date.today.to_s] = today_lines + lines
+
+    last_7_days = (0..6).map { |i| (Date.today - i) }.map(&:to_s)
+    @stats.each_key do |key|
+      @stats.delete(key) unless last_7_days.include?(key)
+    end
+    File.write(@path, @stats.to_json)
+  end
+
+  protected
+
+  def today_lines
+    @stats[Date.today.to_s] || []
+  end
+
+  def load_file
+    @stats = JSON.parse(File.read(@path))
+    @stats ||= {}
+  rescue JSON::ParserError, Errno::ENOENT
+    @stats = {}
+  end
+end

data/lib/rails_log_parser/parser.rb

@@ -23,7 +23,7 @@ class RailsLogParser::Parser
 
   def initialize
     @actions = {}
-    @not_parseable_lines = []
+    @not_parseable_lines = RailsLogParser::NotParseableLines.new
     @heuristic = nil
   end
 
@@ -39,10 +39,11 @@ class RailsLogParser::Parser
       relevant = relevant.select { |a| a.after?(from) }
     end
     summary_output = []
-    if @not_parseable_lines.present?
+    if @not_parseable_lines.lines.present?
       summary_output.push('Not parseable lines:')
-      summary_output += @not_parseable_lines.map { |line| "  #{line}" }
+      summary_output += @not_parseable_lines.lines.map { |line| "  #{line}" }
       summary_output.push("\n\n")
+      @not_parseable_lines.save
     end
 
     %i[warn error fatal].each do |severity|

data/lib/rails_log_parser.rb

@@ -4,11 +4,13 @@ require 'enumerize'
 
 module RailsLogParser
   THRESHOLD_HEURISTIC = 0.02
+  MIN_ACTIONS_HEURISTIC = 100000 # sum of last 10 days
 end
 
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
 require_relative 'rails_log_parser/heuristic_stat_file'
+require_relative 'rails_log_parser/not_parseable_lines'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.8'
+  spec.version       = '0.0.11'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.11
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-10 00:00:00.000000000 Z
+date: 2022-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -71,6 +71,7 @@ files:
 - lib/rails_log_parser/action.rb
 - lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
+- lib/rails_log_parser/not_parseable_lines.rb
 - lib/rails_log_parser/parser.rb
 - lib/rails_log_parser/railtie.rb
 - lib/rails_log_parser/tasks.rb