rails_log_parser 0.0.14 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ecf14cd343a96269393660a0f6daa54bfac43cf45c694840400b295c106b145
-  data.tar.gz: 8690da0687f22e6ee16ecc1cee247c943fb928e28a973b72e45ff141a3792ff8
+  metadata.gz: fe25438fe1aa9e1fe506de50b0d337e26c387648f01aefa8fc558db5e1914bab
+  data.tar.gz: 9ec5eb524baba92586f71dff16fdd31243b5b4a7e68d767a11fb01ec2f40bd3f
 SHA512:
-  metadata.gz: 0ea42a77488c2151e04ef87a0c8d1c8788a3cb1367e8193a67df696622f3319ebdd9d42aa1213d7c4fac7fb00e0a6906401794e36c3e96644e080a80743c3c89
-  data.tar.gz: a43c0c02ce22d35386217c36b7cab9af0cfacde931e6f85931289ab0f890623a6d49b3836695735d7ed4c587efc46708c170052685ffdb1d1a232559c436fe12
+  metadata.gz: 6cb369807f23be8de3ad39d3a616e000d374cd5bb7591b48394c12b38f61b5685fe464bb5712e35d20523089bbd0e69f8e090df7581985df1f30124ab5ba685b
+  data.tar.gz: 5386f801caf9f7a259c5dbc6061793300c6d0347528511f4e0974fe1f64450fd90b92d900730d136f2ce92b8285f9e9c20cdefd9885884e355eae8c0b2ebfcce

data/Gemfile.lock

@@ -1,27 +1,108 @@
 PATH
   remote: .
   specs:
-    rails_log_parser (0.0.13)
+    rails_log_parser (0.0.16)
       enumerize (~> 2.4)
       json (>= 2.0)
+      rails (>= 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.7.3)
+    actioncable (5.2.5)
+      actionpack (= 5.2.5)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailer (5.2.5)
+      actionpack (= 5.2.5)
+      actionview (= 5.2.5)
+      activejob (= 5.2.5)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.2.5)
+      actionview (= 5.2.5)
+      activesupport (= 5.2.5)
+      rack (~> 2.0, >= 2.0.8)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.5)
+      activesupport (= 5.2.5)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.2.5)
+      activesupport (= 5.2.5)
+      globalid (>= 0.3.6)
+    activemodel (5.2.5)
+      activesupport (= 5.2.5)
+    activerecord (5.2.5)
+      activemodel (= 5.2.5)
+      activesupport (= 5.2.5)
+      arel (>= 9.0)
+    activestorage (5.2.5)
+      actionpack (= 5.2.5)
+      activerecord (= 5.2.5)
+      marcel (~> 1.0.0)
+    activesupport (5.2.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    builder (3.2.4)
     concurrent-ruby (1.2.2)
+    crass (1.0.6)
     diff-lcs (1.4.4)
     enumerize (2.5.0)
       activesupport (>= 3.2)
+    erubi (1.10.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
+    loofah (2.9.1)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (1.0.1)
+    method_source (1.0.0)
+    mini_mime (1.1.0)
+    mini_portile2 (2.4.0)
     minitest (5.18.0)
+    nio4r (2.5.7)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (5.2.5)
+      actioncable (= 5.2.5)
+      actionmailer (= 5.2.5)
+      actionpack (= 5.2.5)
+      actionview (= 5.2.5)
+      activejob (= 5.2.5)
+      activemodel (= 5.2.5)
+      activerecord (= 5.2.5)
+      activestorage (= 5.2.5)
+      activesupport (= 5.2.5)
+      bundler (>= 1.3.0)
+      railties (= 5.2.5)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (5.2.5)
+      actionpack (= 5.2.5)
+      activesupport (= 5.2.5)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.19.0, < 2.0)
     rake (12.3.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -36,9 +117,20 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.3)
-    tzinfo (2.0.6)
+    sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.7)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.2)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+    websocket-driver (0.7.3)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
 
 PLATFORMS
   ruby

data/README.md

@@ -31,7 +31,6 @@ Call the rake tasks in cronjobs:
 ```
 LOG_PATH=/srv/rails/log/production.log
 0,20,40 * * * * rake rails_log_parser:parse[22]'      # summary of the last 22 minutes
-59     23 * * * rake rails_log_parser:parse[22,true]' # summary of the last 22 minutes and save and analyse heuristic
 ```
 
 Or use it in your code:
@@ -43,12 +42,33 @@ puts parser.actions.select(&:fatal?).map(&:headline)
 
 ```ruby
 parser = RailsLogParser::Parser.from_file(log_path)
-parser.enable_heuristic(File.dirname(log_path)) # path to save heuristic stats
 print parser.summary(last_minutes: 22)          # print summary for the last 22 minutes
 ```
 
+You can configure filters:
+
+```ruby
+# config/rails_log_parser.rb
+
+RailsLogParser.configure do |parser|
+  parser.ignore_lines = [
+    /Error performing MailerConfigurationJob .+ Net::ReadTimeout/,
+  ]
+end
+```
+
+
 ## Changelog
 
+### 0.0.16
+
+* Remove heuristic
+* Fix last action handling
+
+### 0.0.15
+
+* Filter lines by config file
+
 ### 0.0.14
 
 * Better empty lines handling

data/lib/rails_log_parser/action.rb

@@ -4,10 +4,6 @@ require 'time'
 require 'securerandom'
 
 class RailsLogParser::Action
-  class << self
-    attr_accessor :last
-  end
-
   SEVERITIES = %i[debug info warn error fatal].freeze
   KNOWN_EXCEPTIONS = {
     "Can't verify CSRF token authenticity." => :warn,
@@ -28,7 +24,6 @@ class RailsLogParser::Action
     @id = id
     @messages = []
     @stacktrace = []
-    self.class.last = self
   end
 
   def severity=(value)
@@ -45,6 +40,12 @@ class RailsLogParser::Action
     end
   end
 
+  def ignore?
+    @messages.any? do |message|
+      RailsLogParser.ignore_lines.any? {|ignore| message.match?(ignore) }
+    end
+  end
+
   def headline
     @headline.presence || @messages.first
   end

data/lib/rails_log_parser/parser.rb

@@ -5,7 +5,7 @@ class RailsLogParser::Parser
     attr_writer :log_path
 
     def log_path
-      @log_path || ENV['LOG_PATH']
+      @log_path || ENV['LOG_PATH'] || raise('no log_path given')
     end
 
     def from_file(path)
@@ -19,18 +19,16 @@ class RailsLogParser::Parser
     end
   end
 
-  attr_reader :not_parseable_lines
+  attr_reader :not_parseable_lines, :last_action
 
   def initialize
+    config_file = File.join(Dir.pwd,'config/rails_log_parser.rb')
+    require config_file if File.file?(config_file)
+
     @actions = {}
     @not_parseable_lines = RailsLogParser::NotParseableLines.new
-    @heuristic = nil
   end
 
-  def enable_heuristic(path)
-    @heuristic = path
-    @heuristic_today = RailsLogParser::HeuristicStatFile.new(@heuristic, Date.today).tap { |p| p.write_stats(actions) }
-  end
 
   def summary(last_minutes: nil)
     relevant = actions
@@ -47,7 +45,7 @@ class RailsLogParser::Parser
     end
 
     %i[warn error fatal].each do |severity|
-      selected = relevant.select { |a| a.public_send("#{severity}?") }.reject(&:known_exception?)
+      selected = relevant.select { |a| a.public_send("#{severity}?") }.reject(&:known_exception?).reject(&:ignore?)
       next if selected.blank?
 
       summary_output.push("#{selected.count} lines with #{severity}:")
@@ -55,15 +53,6 @@ class RailsLogParser::Parser
       summary_output.push("\n\n")
     end
 
-    unless @heuristic.nil?
-      stats = RailsLogParser::HeuristicStatFile.build_heuristic(@heuristic, @heuristic_today)
-      if stats.present?
-        summary_output.push("Heuristic match! (threshold: #{RailsLogParser::HeuristicStatFile.heuristic_threshold})")
-        stats.each { |k, v| summary_output.push("- #{k}: #{v.round(4)}") }
-        summary_output.push("\n\n")
-      end
-    end
-
     summary_output.join("\n")
   end
 
@@ -80,6 +69,7 @@ class RailsLogParser::Parser
     @actions[params['id']].severity = params['severity_label']
     @actions[params['id']].datetime = params['datetime']
     @actions[params['id']].add_message(params['message']) unless params['message'].nil?
+    @last_action = @actions[params['id']]
   end
 
   def request(params)
@@ -110,8 +100,4 @@ class RailsLogParser::Parser
     @actions[params['id']] ||= RailsLogParser::Action.new(type, params['id'])
     @actions[params['id']].add_message(params['message'])
   end
-
-  def last_action
-    RailsLogParser::Action.last
-  end
 end

data/lib/rails_log_parser/tasks.rb

@@ -2,9 +2,8 @@
 
 namespace :rails_log_parser do
   desc 'notify about found problems in production.log'
-  task :parse, [:from_minutes, :heuristic] do |_t, args|
+  task :parse, [:from_minutes] do |_t, args|
     parser = RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path)
-    parser.enable_heuristic(File.dirname(RailsLogParser::Parser.log_path)) if args[:heuristic] == 'true'
     print parser.summary(last_minutes: args[:from_minutes])
   end
 end

data/lib/rails_log_parser.rb

@@ -1,16 +1,19 @@
 # frozen_string_literal: true
 
 require 'enumerize'
+require 'active_support/core_ext/module/attribute_accessors'
 
 module RailsLogParser
-  THRESHOLD_HEURISTIC = 0.02
-  MIN_ACTIONS_HEURISTIC = 100000 # sum of last 10 days
+  mattr_accessor :ignore_lines, default: []
+
+  def self.configure
+    yield self
+  end
 end
 
 require_relative 'rails_log_parser/parser'
 require_relative 'rails_log_parser/action'
 require_relative 'rails_log_parser/line'
-require_relative 'rails_log_parser/heuristic_stat_file'
 require_relative 'rails_log_parser/not_parseable_lines'
 
 require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'rails_log_parser'
-  spec.version       = '0.0.14'
+  spec.version       = '0.0.16'
   spec.authors       = ['Georg Limbach']
   spec.email         = ['georg.limbach@lichtbit.com']
 
@@ -24,5 +24,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'enumerize', '~> 2.4'
   spec.add_dependency 'json', '>= 2.0'
+  spec.add_dependency 'rails', '>= 2.1'
   spec.add_development_dependency 'rspec', '>= 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_log_parser
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.16
 platform: ruby
 authors:
 - Georg Limbach
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-27 00:00:00.000000000 Z
+date: 2023-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: enumerize
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -69,7 +83,6 @@ files:
 - Rakefile
 - lib/rails_log_parser.rb
 - lib/rails_log_parser/action.rb
-- lib/rails_log_parser/heuristic_stat_file.rb
 - lib/rails_log_parser/line.rb
 - lib/rails_log_parser/not_parseable_lines.rb
 - lib/rails_log_parser/parser.rb

data/lib/rails_log_parser/heuristic_stat_file.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-RailsLogParser::HeuristicStatFile = Struct.new(:path, :date) do
-  attr_reader :stats
-
-  class << self
-    def build_heuristic(path, today)
-      sums = { actions: 0 }
-      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-        sums[exception.to_sym] = 0
-      end
-      10.times do |i|
-        stats = RailsLogParser::HeuristicStatFile.new(path, today.date - (i + 1)).load_stats
-        sums[:actions] += stats[:actions].to_i
-        RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-          sums[exception.to_sym] += stats.dig(:known_exceptions, exception.to_sym).to_i
-        end
-      end
-      output = {}
-      RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-        next if sums[:actions] < heuristic_min_actions
-
-        quota = sums[exception.to_sym].to_f / sums[:actions]
-        next if quota == 0
-        today_quota = today.rate(exception)
-        next if today_quota == 0
-
-        rate = ((today_quota - quota) / quota) / Math.sqrt(sums[:actions].to_f)
-        output[exception] = rate if rate > heuristic_threshold
-      end
-      output
-    end
-
-    def heuristic_threshold
-      @heuristic_threshold ||= ENV['RAILS_LOG_PARSER_THRESHOLD_HEURISTIC'] || RailsLogParser::THRESHOLD_HEURISTIC
-    end
-
-    def heuristic_min_actions
-      @heuristic_min_actions ||= ENV['RAILS_LOG_PARSER_MIN_ACTIONS_HEURISTIC'] || RailsLogParser::MIN_ACTIONS_HEURISTIC
-    end
-  end
-
-  def write_stats(actions)
-    actions = actions.select { |action| action.datetime.to_date == date }.sort_by(&:datetime)
-    @stats = {
-      actions: actions.count,
-      known_exceptions: {},
-      starts_at: actions.first&.datetime,
-      ends_at: actions.last&.datetime,
-    }
-
-    RailsLogParser::Action::KNOWN_EXCEPTIONS.each_key do |exception|
-      @stats[:known_exceptions][exception.to_sym] = actions.count { |action| action.known_exception?(exception) }
-    end
-
-    delete_old_stats
-    File.write(heuristic_file_path, @stats.to_json)
-  end
-
-  def delete_old_stats
-    last_20_days = (0..19).map { |i| (Date.today - i) }.map { |date| File.join(path, "heuristic_stats_#{date}.json") }
-    Dir[File.join(path, 'heuristic_stats_*.json')].reject { |file| last_20_days.include?(file) }.each do |file|
-      File.unlink(file)
-    end
-  end
-
-  def load_stats
-    @stats = JSON.parse(File.read(heuristic_file_path), symbolize_names: true) if File.file?(heuristic_file_path)
-    @stats ||= {}
-  rescue JSON::ParserError
-    @stats = {}
-  end
-
-  def heuristic_file_path
-    @heuristic_file_path ||= File.join(path, "heuristic_stats_#{date}.json")
-  end
-
-  def rate(exception)
-    return 0 if stats[:actions] == 0
-
-    stats.dig(:known_exceptions, exception.to_sym).to_f / stats[:actions]
-  end
-end