log_sense 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43c38b704af22a839fedec504755be79abf4307c1183efe65ee7323984c4d6bc
-  data.tar.gz: ed1b73c4b9ccc5e6ffcae7682e67124aa82dba1d9fde9ab625764207893a11dd
+  metadata.gz: 01b7c715a9aac207ec39985ba5791c307e64784ae1c5c3f5f5a4e0b124a796af
+  data.tar.gz: 7f1881b1a621ea2d1b2f3f29052d4a82016d23bcb6c0d702712b249308bbf2cc
 SHA512:
-  metadata.gz: 19857e10b0f5b72afdb491245101050a3348e267d6a9409416ab478e72b441ed8b559280f492f1c761753cbfd71abdafd5184f60095c53ca1bc0640a83c22f4a
-  data.tar.gz: 3f049b2b6756bfe9ef872021f6b34263736b12eb75ec2b06f086387550fbd87efe367ccc4348070cb492f5870b286c626fe6244ab19ea01f74684d301175b32b
+  metadata.gz: 0e8ccec75c78ca240320c3be45cf47779879eef0cd7033fee0e623647b29c744794e19e9f4b4f2e444e9b474b90b017ab10b60bea6661f92aaf156923aff839e
+  data.tar.gz: b70eb6964f19f0e7272b3953a4df5b124ea95c1d75ea6fd1a33f9ffe0ad04f6ddb2c7e9e177ccd19dded06a94ae159b3333fca9fadb1e85c279b87cccf78717c

data/.gitignore

@@ -8,3 +8,6 @@
 /tmp/
 node_modules
 *~
+TAGS
+sample_logs
+ip_locations/dbip-country-lite-*.csv*

data/CHANGELOG.org

@@ -2,6 +2,12 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.4.0
+
+- Updates log_parser to match changes in log lines introduced from Rails 7
+  (or, in other words, fixes a bug which caused log_sense to miss events
+  from logs produced by Rails >= 7 applications)
+  
 * 2.3.1
 
 - Updates geoip db

data/Gemfile.lock

@@ -3,8 +3,10 @@ PATH
   specs:
     log_sense (2.3.1)
       browser (~> 6.0.0)
+      csv
       ipaddr (~> 1.2.0)
       iso_country_codes (~> 0.7.0)
+      ostruct
       sqlite3 (~> 2.6.0)
       terminal-table (~> 4.0.0)
 
@@ -18,6 +20,7 @@ GEM
     bundler-audit (0.9.2)
       bundler (>= 1.2.0, < 3)
       thor (~> 1.0)
+    csv (3.3.4)
     date (3.4.1)
     debug (1.10.0)
       irb (~> 1.10)
@@ -34,6 +37,7 @@ GEM
     lint_roller (1.1.0)
     mini_portile2 (2.8.9)
     minitest (5.25.5)
+    ostruct (0.6.1)
     parallel (1.27.0)
     parser (3.3.8.0)
       ast (~> 2.4.1)

data/README.org

@@ -4,8 +4,8 @@
 
 * Introduction
 
-LogSense generates reports and statistics from Ruby on Rails and Apache/Nginx
-log files.
+LogSense analyzes Apache and Rails log files, generating reports and statistics
+about usage and performace.
 
 Main features:
 
@@ -22,10 +22,11 @@ Main features:
   self polls and crawlers.
 - Reports can be generated in HTML, txt, ufw, and SQLite.  HTML reports are
   responsive and come with dark and light theme.
+- It works with Rails' vanilla log format (no need to format the log)
 
 LogSense is Written in Ruby, it runs from the command line, it is fast, and it
 can be installed on any system with a relatively recent version of Ruby.  We
-use it with Ruby 3.1.4 and 3.3.0.
+started using it with Ruby 3.1.4 and we are now at version 3.4.
 
 It is fast. On a ThinkPad P16, a 277M log file is parsed in 15 seconds,
 processing, that is, about 7740 events per second; a 569M log file is parsed in

data/lib/log_sense/rails/log_parser.rb

@@ -33,6 +33,9 @@ module LogSense
           duration_views_ms FLOAT,
           duration_ar_ms FLOAT,
           allocations INTEGER,
+          queries INTEGER,
+          cached_queries INTEGER,
+          gc_duration INTEGER,
           comment TEXT,
           source_file TEXT,
           line_number INTEGER
@@ -180,9 +183,16 @@ module LogSense
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Processing by CONTROLLER as FORMAT
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Parameters: JSON
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Rendered VIEW within LAYOUT (Duration: DURATION | Allocations: ALLOCATIONS)
+        #
+        # For rails_6:
+        #
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Completed STATUS STATUS_STRING in DURATION (Views: DURATION | ActiveRecord: DURATION | Allocations: NUMBER)
         #
-        # and they appears in the order shown above: started, processing, ...
+        # For rails_7:
+        #
+        # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Completed STATUS STATUS_STRING in DURATION (Views: DURATION | ActiveRecord: DURATION (N queries, M cached) | GC: DURATION)
+        #
+        # and they appear in the order shown above: started, processing, ...
         #
         # Different requests might be interleaved, of course
         #
@@ -434,6 +444,7 @@ module LogSense
       STATUS = '(?<status>[0-9]+)'
       STATUS_IN_WORDS = '(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)'
       MSECS = '[0-9.]+'
+      INT = '[0-9]+'
 
       # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
       STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Started #{VERB} "#{URL}" for #{IP} at/o
@@ -455,7 +466,21 @@ module LogSense
       # I, [2021-10-19T08:16:34.712331 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
       # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
       # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
-      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/o
+
+      # REGEXP Fragments
+
+      # views is optional
+      # In strings I need to properly escape \ or it will be mis-interpreted
+      VIEWS = "(Views: (?<views>#{MSECS})ms \\| )?"
+      # active records has two formats (according to Rails version)
+      ACTIVE_RECORDS_BASE = "ActiveRecord: (?<arec>#{MSECS})ms"
+      ACTIVE_RECORDS_QUERIES = "( \\((?<queries>#{INT}) queries, (?<cached_queries>#{INT}) cached\\))?"
+      ACTIVE_RECORDS = "#{ACTIVE_RECORDS_BASE}#{ACTIVE_RECORDS_QUERIES}"
+      # older rails versions have ALLOCATIONS, newer have GC
+      ALLOCATIONS = " \\| Allocations: (?<alloc>#{INT})"
+      GC = " \\| GC: (?<gc_duration>#{MSECS})ms"
+
+      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \(#{VIEWS}#{ACTIVE_RECORDS}(#{ALLOCATIONS}|#{GC})?\)/o
 
       def match_and_process_completed(line)
         matchdata = (COMPLETED_REGEXP.match line)
@@ -469,7 +494,10 @@ module LogSense
             duration_total_ms: matchdata[:total],
             duration_views_ms: matchdata[:views],
             duration_ar_ms: matchdata[:arec],
-            allocations: matchdata[:alloc],
+            allocations: (matchdata[:alloc] || -1),
+            queries: (matchdata[:queries] || -1),
+            cached_queries: (matchdata[:cached_queries] || -1),
+            gc_duration: (matchdata[:gc_duration] || -1),
             comment: ""
           }
         end

data/lib/log_sense/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LogSense
-  VERSION = "2.3.1"
+  VERSION = "2.4.0"
 end

data/log_sense.gemspec

@@ -6,8 +6,18 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adolfo Villafiorita"]
   spec.email         = ["adolfo@shair.tech"]
 
-  spec.summary       = %q{Generate analytics for Rails and Apache/Nginx log file.}
-  spec.description   = %q{Generate analytics in HTML, txt, and SQLite format for Rails and Apache/Nginx log files.}
+  spec.summary       = %q{Analyze Rails and Apache/Nginx log file, generating statistics and reports.}
+  spec.description   = %q{log_sense is a command-line command which analyzes Rails and Nginx logs and
+    provides detailed data about accesses, performances, and errors.  It is a
+    one stop shop solution for analyzing your Rails application in production.
+
+    Simple to use and working offline, it does not require any modification to
+    your app (e.g., no need to change the log format, no cookies, no JavaScript
+    code, no infrastructure to setup).
+
+    We use it to monitor our applications in production, using a cron job which
+    pulls the logs and calls log_sense to generate HTML reports.
+  }
   spec.homepage      = "https://github.com/shair-tech/log_sense/"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.6.9")
@@ -26,7 +36,9 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-
+  
+  spec.add_dependency "ostruct"
+  spec.add_dependency "csv"
   spec.add_dependency "browser", "~> 6.0.0"
   spec.add_dependency "ipaddr", "~> 1.2.0"
   spec.add_dependency "iso_country_codes", "~> 0.7.0" 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.4.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
@@ -9,6 +9,34 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: browser
   requirement: !ruby/object:Gem::Requirement
@@ -135,8 +163,14 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Generate analytics in HTML, txt, and SQLite format for Rails and Apache/Nginx
-  log files.
+description: "log_sense is a command-line command which analyzes Rails and Nginx logs
+  and\n    provides detailed data about accesses, performances, and errors.  It is
+  a\n    one stop shop solution for analyzing your Rails application in production.\n\n
+  \   Simple to use and working offline, it does not require any modification to\n
+  \   your app (e.g., no need to change the log format, no cookies, no JavaScript\n
+  \   code, no infrastructure to setup).\n\n    We use it to monitor our applications
+  in production, using a cron job which\n    pulls the logs and calls log_sense to
+  generate HTML reports.\n  "
 email:
 - adolfo@shair.tech
 executables:
@@ -218,5 +252,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.7
 specification_version: 4
-summary: Generate analytics for Rails and Apache/Nginx log file.
+summary: Analyze Rails and Apache/Nginx log file, generating statistics and reports.
 test_files: []