rails_log_parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a7d5317b56ac5adc2cd38dae3ec083ab684960a1ce622821140c8b1d7ecfe33c
+  data.tar.gz: 12f6efea87dfc891e31eadadbb1e6a5993b1c5b8c5fee0ea648246415b55c167
+SHA512:
+  metadata.gz: 2a995df0bc41c2e81ef0b0cbd283592a24a17f030f1b02fb868d50d1b79f0982643187e9013fd9cddb2539dc9fe7a82a1e290198a5214e1373c735c63b9d305a
+  data.tar.gz: f999faf6b4905bd5ee0191d0a6402e456928836bb296970407c00bf91635e575fb147fa88b064de272882871a301da467134bc64399c6868c9045b2cfb711025

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rails_log_parser.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    rails_log_parser (0.0.1)
+      enumerize (~> 2.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    concurrent-ruby (1.1.9)
+    diff-lcs (1.4.4)
+    enumerize (2.4.0)
+      activesupport (>= 3.2)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    rake (12.3.3)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rails_log_parser!
+  rake (~> 12.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Georg Limbach
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# RailsLogParser
+
+This gem helps you to quick analyse your server logs without external monitor servers. Simple applications with a low number of requests cause only a low number of errors. So you can call a cronjob to analyse your logs periodly. If the summary is not empty your server will email you the result automaticly.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rails_log_parser'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install rails_log_parser
+
+## Usage
+
+Sets your server log path:
+
+```ruby
+RailsLogParser::Parser.log_path = Rails.root.join('log/production.log')
+```
+
+Call the rake tasks in cronjobs:
+
+```
+0,20,40 * * * * rake rails_log_parser:parse[22]' # summary of the last 22 minutes
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Lichtbit/rails_log_parser.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/lib/rails_log_parser/action.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require 'time'
+require 'securerandom'
+
+class RailsLogParser::Action
+  class << self
+    attr_accessor :last
+  end
+
+  SEVERITIES = %i[debug info warn error fatal].freeze
+  KNOWN_EXCEPTIONS = {
+    'ActiveRecord::RecordNotFound' => :fatal,
+    'ActionController::RoutingError' => :fatal,
+    "Can't verify CSRF token authenticity." => :warn,
+  }.freeze
+
+  extend Enumerize
+  enumerize :severity, in: SEVERITIES, predicates: true
+  enumerize :type, in: %i[request without_request delayed_job active_job], predicates: true
+
+  def initialize(type, id)
+    self.type = type
+    @id = id
+    @messages = []
+    @stacktrace = []
+    self.class.last = self
+  end
+
+  def severity=(value)
+    value = value.downcase.to_sym
+    return unless severity.nil? || SEVERITIES.index(severity.to_sym) < SEVERITIES.index(value)
+
+    super(value)
+    @headline = nil
+  end
+
+  def known_exception?
+    @messages.any? do |message|
+      KNOWN_EXCEPTIONS.any? { |e, s| message.include?(e) && severity == s }
+    end
+  end
+
+  def headline
+    @headline.presence || @messages.first
+  end
+
+  def datetime=(value)
+    @datetime ||= Time.parse(value)
+  end
+
+  def after?(datetime)
+    @datetime > datetime
+  end
+
+  def add_message(value)
+    @messages.push(value)
+    @headline = value if @headline.nil?
+  end
+
+  def add_stacktrace(value)
+    @stacktrace.push(value)
+  end
+end

data/lib/rails_log_parser/line.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+class RailsLogParser::Line
+  attr_reader :parser, :line
+
+  def initialize(parser, line)
+    @parser = parser
+    @line = line.strip
+
+    parse
+  end
+
+  def parse
+    return if line.empty?
+
+    # empty log line
+    match = line.match(/
+      \A(?<severity_id>[DIWEFU]),\s                 # I,
+      \[(?<datetime>[^\]]+)\s\#(?<pid>\d+)\]\s+     # [2021-11-26T13:11:01.255168 #10158]
+      (?<severity_label>[A-Z]+)\s+--\s+[^:]*:\s+    # INFO -- :
+      \[(?<id>[a-f0-9\-]{36})\]\s*                  # [b42b65ab-7985-4bc2-a5b5-1fb23e6ad940]
+      \z/x)
+    return if match
+
+    # normal log line
+    match = line.match(/
+      \A(?<severity_id>[DIWEFU]),\s                 # I,
+      \[(?<datetime>[^\]]+)\s\#(?<pid>\d+)\]\s+     # [2021-11-26T13:11:01.255168 #10158]
+      (?<severity_label>[A-Z]+)\s+--\s+[^:]*:\s+    # INFO -- :
+      \[(?<id>[a-f0-9\-]{36})\]\s                   # [b42b65ab-7985-4bc2-a5b5-1fb23e6ad940]
+      (?<message>.*)                                # Processing by Public::Controller
+      \z/x)
+    if match
+      parser.request(match)
+      return
+    end
+
+    match = line.match(/
+      \A(?<severity_id>[DIWEFU]),\s                 # I,
+      \[(?<datetime>[^\]]+)\s\#(?<pid>\d+)\]\s+     # [2021-11-26T13:11:01.255168 #10158]
+      (?<severity_label>[A-Z]+)\s+--\s+[^:]*:\s+    # INFO -- :
+      \[ActiveJob\]                                 # [ActiveJob]
+      (?:\s\[[^\]]+\])?                             # [ActionMailer::Parameterized::DeliveryJob]
+      \s\[(?<id>[^\]]+)\]\s                         # [09f4c08a-b92e-42e3-9046-7effcf87aa2f]
+      (?<message>.*)                                # Performing ActionMailer::Parameterized::DeliveryJob
+      \z/x)
+    if match
+      parser.active_job(match)
+      return
+    end
+
+    match = line.match(/
+      \A(?<severity_id>[DIWEFU]),\s                 # I,
+      \[(?<datetime>[^\]]+)\s\#(?<pid>\d+)\]\s+     # [2021-11-26T13:11:01.255168 #10158]
+      (?<severity_label>[A-Z]+)\s+--\s+[^:]*:\s     # INFO -- :
+      (?<datetime2>[^\s]+)\s                        # 2021-11-26T13:51:30+0000:
+      \[Worker[^\]]+\]\s                            # [Worker(delayed_job host:production pid:10411)]
+      (?<message>.*)                                # Job GeoLocationSearch
+      (?<id>\(id=\d+\)\s\([^)]+\))                  # (id=148781) (queue=default)
+      (?<message2>.*)                               # RUNNING
+      \z/x)
+    if match
+      parser.delayed_job(match)
+      return
+    end
+
+    match = line.match(/
+      \A(?<severity_id>[DIWEFU]),\s                 # I,
+      \[(?<datetime>[^\]]+)\s\#(?<pid>\d+)\]\s+     # [2021-11-26T13:11:01.255168 #10158]
+      (?<severity_label>[A-Z]+)\s+--\s+[^:]*:\s     # INFO -- :
+      (?<datetime2>[^\s]+)\s                        # 2021-11-26T13:51:30+0000:
+      \[Worker[^\]]+\]\s                            # [Worker(delayed_job host:production pid:10411)]
+      (?:\d+\sjobs\sprocessed)|(?:Starting\sjob)    # 19 jobs processed
+      (?:.*)                                        # at 2.8816
+      \z/x)
+    return if match
+
+    # Warning or Message without request
+    match = line.match(/
+      \A(?<severity_id>[DIWEFU]),\s                 # I,
+      \[(?<datetime>[^\]]+)\s\#(?<pid>\d+)\]\s+     # [2021-11-26T13:11:01.255168 #10158]
+      (?<severity_label>[A-Z]+)\s+--\s+[^:]*:\s     # INFO -- :
+      (?<message>.*)                                # Creating scope :for_tipster. Overwriting existing...
+      \z/x)
+    if match
+      parser.without_request(match)
+      return
+    end
+
+    match = line.match(/
+      \A\[(?<id>[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12})\]\s # [UUID]
+      (?<message>.*)
+      \z/x)
+    if match
+      parser.add_message(match)
+      return
+    end
+
+    if parser.last_action&.error?
+      parser.last_action.add_stacktrace(line)
+      return
+    end
+
+    parser.not_parseable_lines.push(line)
+  end
+end

data/lib/rails_log_parser/parser.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+class RailsLogParser::Parser
+  class << self
+    attr_writer :log_path
+
+    def log_path
+      @log_path || ENV['LOG_PATH']
+    end
+
+    def from_file(path)
+      parser = new
+      File.open(path, 'r') do |handle|
+        while (line = handle.gets)
+          parser.puts(line)
+        end
+      end
+      parser
+    end
+  end
+
+  attr_reader :not_parseable_lines
+
+  def initialize
+    @actions = {}
+    @not_parseable_lines = []
+  end
+
+  def summary(last_minutes: nil)
+    relevant = actions
+    if last_minutes.present?
+      from = last_minutes.to_i.minutes.ago
+      relevant = relevant.select { |a| a.after?(from) }
+    end
+    @summary_output = []
+    if @not_parseable_lines.present?
+      @summary_output.push('Not parseable lines:')
+      @summary_output += @not_parseable_lines.map { |line| "  #{line}" }
+      @summary_output.push("\n\n")
+    end
+
+    %i[warn error fatal].each do |severity|
+      selected = relevant.select { |a| a.public_send("#{severity}?") }.reject(&:known_exception?)
+      next if selected.blank?
+
+      @summary_output.push("#{selected.count} lines with #{severity}:")
+      @summary_output += selected.map(&:headline).map { |line| "  #{line}" }
+      @summary_output.push("\n\n")
+    end
+
+    @summary_output.join("\n")
+  end
+
+  def actions
+    @actions.values
+  end
+
+  def puts(line)
+    RailsLogParser::Line.new(self, line.encode('UTF-8', invalid: :replace))
+  end
+
+  def action(type, params)
+    @actions[params['id']] ||= RailsLogParser::Action.new(type, params['id'])
+    @actions[params['id']].severity = params['severity_label']
+    @actions[params['id']].datetime = params['datetime']
+    @actions[params['id']].add_message(params['message'])
+  end
+
+  def request(params)
+    action(:request, params)
+  end
+
+  def without_request(params)
+    params = params.named_captures
+    params['id'] = SecureRandom.uuid
+    action(:without_request, params)
+  end
+
+  def active_job(params)
+    action(:active_job, params)
+  end
+
+  def delayed_job(params)
+    action(:delayed_job, params)
+  end
+
+  def add_message(params)
+    @actions[params['id']] ||= RailsLogParser::Action.new(type, params['id'])
+    @actions[params['id']].add_message(params['message'])
+  end
+
+  def last_action
+    RailsLogParser::Action.last
+  end
+end

data/lib/rails_log_parser/railtie.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'rails_log_parser'
+require 'rails'
+
+class RailsLogParser::Railtie < Rails::Railtie
+  rake_tasks do
+    load 'rails_log_parser/tasks.rb'
+  end
+end

data/lib/rails_log_parser/tasks.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+namespace :rails_log_parser do
+  desc 'notify about found problems in production.log'
+  task :parse, [:from_minutes] do |_t, args|
+    print RailsLogParser::Parser.from_file(RailsLogParser::Parser.log_path).summary(last_minutes: args[:from_minutes])
+  end
+end

data/lib/rails_log_parser.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'enumerize'
+
+module RailsLogParser
+end
+
+require_relative 'rails_log_parser/parser'
+require_relative 'rails_log_parser/action'
+require_relative 'rails_log_parser/line'
+
+require 'rails_log_parser/railtie' if defined?(Rails::Railtie)

data/rails_log_parser.gemspec

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.name          = 'rails_log_parser'
+  spec.version       = '0.0.1'
+  spec.authors       = ['Georg Limbach']
+  spec.email         = ['georg.limbach@lichtbit.com']
+
+  spec.summary       = 'Simple and fast analysing of default rails logs'
+  spec.description   = 'If you want to get an email with errors and fatal log lines you can use this gem.'
+  spec.homepage      = 'https://github.com/Lichtbit/rails_log_parser'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/Lichtbit/rails_log_parser'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'enumerize', '~> 2.4'
+  spec.add_development_dependency 'rspec'
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: rails_log_parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Georg Limbach
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: enumerize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: If you want to get an email with errors and fatal log lines you can use
+  this gem.
+email:
+- georg.limbach@lichtbit.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/rails_log_parser.rb
+- lib/rails_log_parser/action.rb
+- lib/rails_log_parser/line.rb
+- lib/rails_log_parser/parser.rb
+- lib/rails_log_parser/railtie.rb
+- lib/rails_log_parser/tasks.rb
+- rails_log_parser.gemspec
+homepage: https://github.com/Lichtbit/rails_log_parser
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/Lichtbit/rails_log_parser
+  source_code_uri: https://github.com/Lichtbit/rails_log_parser
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Simple and fast analysing of default rails logs
+test_files: []