rails_log_parser 0.0.6 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f608985d7c53f5ac09850df6c94d63437c5c1ecec18390072e83b4751a57e16
-  data.tar.gz: 68606ff5cea0948ad95cdd6f0ae583374b73654c013e95807dc0242070291879
+  metadata.gz: 96d17b92a0e47f95f636d1f1006e5a6e32573c08d22c26269a80b63f8fc379c5
+  data.tar.gz: 3693d726cd73d53479917370c967fb2d028a5ad9ef4cc27442e413eabb84f88c
 SHA512:
-  metadata.gz: 7351dc20aa2fc84268825e955ef6b8ffe924571013c27e6b0728555442aa205c10600a5266c153387109fb9fe7f6bc03d2657fecc50956622a2ac7eb9e6339bc
-  data.tar.gz: eb0689bc7193fc130aae3cc40dd68ad5fc865add71696bd9a044442ef3094e61a26966f0044967cf8939365f0b03cf64d6b9b96ee5dc559236f0c80eb5770634
+  metadata.gz: 389480dd2c6b6f6dd8c732b7bc07c458df3190a7fc1f56253adb8a780a1717efebd17947c5e4c8050b912f6838487a01c65737b45feb51867ce4830beeda5057
+  data.tar.gz: f834c008d3a24a589a51f8ba5cfa70b3bcdfb372987df5875d30a0402848cde4184531dd4fb5d9e82f5979d284c23d53f6bb090ab276f3c221719f0563d0d593

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.6)
+    rails_log_parser (0.0.9)
       enumerize (~> 2.4)
-      json
+      json (>= 2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (6.1.4.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -16,12 +16,12 @@ GEM
       zeitwerk (~> 2.3)
     concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
-    enumerize (2.4.0)
+    enumerize (2.5.0)
       activesupport (>= 3.2)
     i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    json (2.3.1)
-    minitest (5.14.4)
+    json (2.6.1)
+    minitest (5.15.0)
     rake (12.3.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -38,7 +38,7 @@ GEM
     rspec-support (3.10.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.3)
 
 PLATFORMS
   ruby

data/README.md

@@ -49,6 +49,20 @@ print parser.summary(last_minutes: 22)          # print summary for the last 22
 
 ## Changelog
 
+### 0.0.9
+
+* Message not parseable lines only once a day
+* Delete old heuristic stat files automaticly
+
+### 0.0.8
+
+* Adding `ActionController::UnfilteredParameters` as known exceptions
+* Adjust heuristic rate for better matching
+
+### 0.0.7
+
+* Remove empty lines on summary without report
+
 ### 0.0.6
 
 * Adding heuristic to rate known exceptions

data/lib/rails_log_parser/action.rb

@@ -14,6 +14,7 @@ class RailsLogParser::Action
     'ActionController::RoutingError' => :fatal,
     "Can't verify CSRF token authenticity." => :warn,
     'ActionController::InvalidAuthenticityToken' => :fatal,
+    'ActionController::UnfilteredParameters' => :fatal,
   }.freeze
 
   extend Enumerize

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -11,7 +11,7 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
       RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
         sums[exception.to_sym] = 0
       end
-      7.times do |i|
+      10.times do |i|
         stats = RailsLogParser::HeuristicStatFile.new(path, today.date - (i + 1)).load_stats
         sums[:actions] += stats[:actions].to_i
         RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
@@ -49,9 +49,18 @@ RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
     RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
       @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
     end
+
+    delete_old_stats
     File.write(heuristic_file_path, @stats.to_json)
   end
 
+  def delete_old_stats
+    last_20_days = (0..19).map { |i| (Date.today - i) }.map { |date| File.join(path, "heuristic_stats_#{date}.json") }
+    Dir[File.join(path, 'heuristic_stats_*.json')].reject { |file| last_20_days.include?(file) }.each do |file|
+      File.unlink(file)
+    end
+  end
+
   def load_stats
     @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
     @stats ||= {}

data/lib/rails_log_parser/not_parseable_lines.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'json'
+
+class RailsLogParser::NotParseableLines
+  attr_reader :lines
+
+  def initialize
+    @lines = []
+    @path = File.join(File.dirname(RailsLogParser::Parser.log_path), 'not_parseable_lines.json')
+    load_file
+  end
+
+  def push(line)
+    @lines.push(line) unless today_lines.include?(line)
+  end
+
+  def save
+    @stats[Date.today.to_s] = today_lines + lines
+
+    last_7_days = (0..6).map { |i| (Date.today - i) }.map(&:to_s)
+    @stats.each_key do |key|
+      @stats.delete(key) unless last_7_days.include?(key)
+    end
+    File.write(@path, @stats.to_json)
+  end
+
+  protected
+
+  def today_lines
+    @stats[Date.today.to_s] || []
+  end
+
+  def load_file
+    @stats = JSON.parse(File.read(@path))
+    @stats ||= {}
+  rescue JSON::ParserError, Errno::ENOENT
+    @stats = {}
+  end
+end

data/lib/rails_log_parser/parser.rb

@@ -23,7 +23,7 @@ class RailsLogParser::Parser
 
   def initialize
     @actions = {}
-    @not_parseable_lines = []
+    @not_parseable_lines = RailsLogParser::NotParseableLines.new
     @heuristic = nil
   end
 
@@ -39,10 +39,11 @@ class RailsLogParser::Parser
       relevant = relevant.select { |a| a.after?(from) }
     end
     summary_output = []
-    if @not_parseable_lines.present?
+    if @not_parseable_lines.lines.present?
       summary_output.push('Not parseable lines:')
-      summary_output += @not_parseable_lines.map { |line| "  #{line}" }
+      summary_output += @not_parseable_lines.lines.map { |line| "  #{line}" }
       summary_output.push("\n\n")
+      @not_parseable_lines.save
     end
 
     %i[warn error fatal].each do |severity|
@@ -57,10 +58,10 @@ class RailsLogParser::Parser
     unless @heuristic.nil?
       stats = RailsLogParser::HeuristicStatFile.build_heuristic(@heuristic, @heuristic_today)
       if stats.present?
-        summary_output.push('Heuristic match!')
-        stats.each { |k, v| summary_output.push("- #{k}: #{v}") }
+        summary_output.push("Heuristic match! (threshold: #{RailsLogParser::HeuristicStatFile.heuristic_threshold})")
+        stats.each { |k, v| summary_output.push("- #{k}: #{v.round(4)}") }
+        summary_output.push("\n\n")
       end
-      summary_output.push("\n\n")
     end
 
     summary_output.join("\n")

data/lib/rails_log_parser.rb

@@ -3,12 +3,13 @@
 require 'enumerize'
 
 module RailsLogParser
-  THRESHOLD_HEURISTIC = 0.01
+  THRESHOLD_HEURISTIC = 0.02
 end
 
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
 require_relative 'rails_log_parser/heuristic_stat_file'
+require_relative 'rails_log_parser/not_parseable_lines'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.6'
+  spec.version       = '0.0.9'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 
@@ -23,6 +23,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'enumerize', '~> 2.4'
-  spec.add_dependency 'json'
-  spec.add_development_dependency 'rspec'
+  spec.add_dependency 'json', '>= 2.0'
+  spec.add_development_dependency 'rspec', '>= 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.9
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-15 00:00:00.000000000 Z
+date: 2022-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -30,28 +30,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 description: If you want to get an email with errors and fatal log lines you can use
   this gem.
 email:
@@ -71,6 +71,7 @@ files:
 - lib/rails_log_parser/action.rb
 - lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
+- lib/rails_log_parser/not_parseable_lines.rb
 - lib/rails_log_parser/parser.rb
 - lib/rails_log_parser/railtie.rb
 - lib/rails_log_parser/tasks.rb