log_sense 2.3.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a9992db2fa42d1de3dac0bacbeed543a097e351c8bc9999c60febb3f67714a9
-  data.tar.gz: 06b9ed1329cbf3fdfdf889d154576dff85f16178e817b29fffede535eb1f12be
+  metadata.gz: 01b7c715a9aac207ec39985ba5791c307e64784ae1c5c3f5f5a4e0b124a796af
+  data.tar.gz: 7f1881b1a621ea2d1b2f3f29052d4a82016d23bcb6c0d702712b249308bbf2cc
 SHA512:
-  metadata.gz: 0510a72e71e67e7bba948219f3ab35e7422c95a5a167815ccd30f589c172f19f7704e6e583364ac2493a7ff32c18df1316ef36d05f7e0346f582516b62d9f806
-  data.tar.gz: a374eea7ae662ac5be2458620baa4276bd7957871ccea67e82ba2987f941257ec2ffd1d1e7b62efd764093204976759470ccf7ca08a8152ecf1776d5eb023499
+  metadata.gz: 0e8ccec75c78ca240320c3be45cf47779879eef0cd7033fee0e623647b29c744794e19e9f4b4f2e444e9b474b90b017ab10b60bea6661f92aaf156923aff839e
+  data.tar.gz: b70eb6964f19f0e7272b3953a4df5b124ea95c1d75ea6fd1a33f9ffe0ad04f6ddb2c7e9e177ccd19dded06a94ae159b3333fca9fadb1e85c279b87cccf78717c

data/.gitignore

@@ -8,3 +8,6 @@
 /tmp/
 node_modules
 *~
+TAGS
+sample_logs
+ip_locations/dbip-country-lite-*.csv*

data/CHANGELOG.org

@@ -2,6 +2,17 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.4.0
+
+- Updates log_parser to match changes in log lines introduced from Rails 7
+  (or, in other words, fixes a bug which caused log_sense to miss events
+  from logs produced by Rails >= 7 applications)
+  
+* 2.3.1
+
+- Updates geoip db
+- Updates dependencies    
+
 * 2.3.0
 
 - Adds average response time by day

data/Gemfile.lock

@@ -1,83 +1,95 @@
 PATH
   remote: .
   specs:
-    log_sense (2.2.1)
-      browser (~> 5.3.0)
+    log_sense (2.3.1)
+      browser (~> 6.0.0)
+      csv
       ipaddr (~> 1.2.0)
       iso_country_codes (~> 0.7.0)
-      sqlite3 (~> 2.0.0)
-      terminal-table (~> 3.0.0)
+      ostruct
+      sqlite3 (~> 2.6.0)
+      terminal-table (~> 4.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.2)
-    browser (5.3.1)
+    ast (2.4.3)
+    browser (6.0.0)
     bundle-audit (0.1.0)
       bundler-audit
     bundler-audit (0.9.2)
       bundler (>= 1.2.0, < 3)
       thor (~> 1.0)
-    debug (1.9.2)
+    csv (3.3.4)
+    date (3.4.1)
+    debug (1.10.0)
       irb (~> 1.10)
       reline (>= 0.3.8)
-    io-console (0.7.2)
-    ipaddr (1.2.6)
-    irb (1.14.0)
+    io-console (0.8.0)
+    ipaddr (1.2.7)
+    irb (1.15.2)
+      pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     iso_country_codes (0.7.8)
-    json (2.7.2)
-    language_server-protocol (3.17.0.3)
-    mini_portile2 (2.8.7)
-    minitest (5.24.1)
-    parallel (1.26.3)
-    parser (3.3.4.2)
+    json (2.12.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    mini_portile2 (2.8.9)
+    minitest (5.25.5)
+    ostruct (0.6.1)
+    parallel (1.27.0)
+    parser (3.3.8.0)
       ast (~> 2.4.1)
       racc
-    psych (5.1.2)
+    pp (0.6.2)
+      prettyprint
+    prettyprint (0.2.0)
+    prism (1.4.0)
+    psych (5.2.6)
+      date
       stringio
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.2.1)
-    rdoc (6.7.0)
+    rdoc (6.13.1)
       psych (>= 4.0.0)
-    regexp_parser (2.9.2)
-    reline (0.5.9)
+    regexp_parser (2.10.0)
+    reline (0.6.1)
       io-console (~> 0.5)
-    rexml (3.3.6)
-      strscan
-    rubocop (1.65.1)
+    rubocop (1.75.5)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.4, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.32.1)
-      parser (>= 3.3.1.0)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
     ruby-progressbar (1.13.0)
-    sqlite3 (2.0.3)
+    sqlite3 (2.6.0)
       mini_portile2 (~> 2.8.0)
-    stringio (3.1.1)
-    strscan (3.1.0)
-    terminal-table (3.0.2)
-      unicode-display_width (>= 1.1.1, < 3)
-    thor (1.3.1)
-    unicode-display_width (2.5.0)
+    stringio (3.1.7)
+    terminal-table (4.0.0)
+      unicode-display_width (>= 1.1.1, < 4)
+    thor (1.3.2)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundle-audit
-  debug (~> 1.9.0)
+  debug
   log_sense!
-  minitest (~> 5.24.0)
+  minitest
   rake (~> 13.0)
   rubocop
 

data/README.org

@@ -4,8 +4,8 @@
 
 * Introduction
 
-LogSense generates reports and statistics from Ruby on Rails and Apache/Nginx
-log files.
+LogSense analyzes Apache and Rails log files, generating reports and statistics
+about usage and performace.
 
 Main features:
 
@@ -22,10 +22,11 @@ Main features:
   self polls and crawlers.
 - Reports can be generated in HTML, txt, ufw, and SQLite.  HTML reports are
   responsive and come with dark and light theme.
+- It works with Rails' vanilla log format (no need to format the log)
 
 LogSense is Written in Ruby, it runs from the command line, it is fast, and it
 can be installed on any system with a relatively recent version of Ruby.  We
-use it with Ruby 3.1.4 and 3.3.0.
+started using it with Ruby 3.1.4 and we are now at version 3.4.
 
 It is fast. On a ThinkPad P16, a 277M log file is parsed in 15 seconds,
 processing, that is, about 7740 events per second; a 569M log file is parsed in

data/lib/log_sense/options/parser.rb

@@ -139,7 +139,7 @@ module LogSense
 
           opts.on("-v", "--version", "Prints version information") do
             puts "log_sense version #{LogSense::VERSION}"
-            puts "Copyright (C) 2021-2024 Shair.Tech"
+            puts "Copyright (C) 2021-2025 Shair.Tech"
             puts "Distributed under the terms of the MIT license"
             exit
           end

data/lib/log_sense/rails/log_parser.rb

@@ -33,6 +33,9 @@ module LogSense
           duration_views_ms FLOAT,
           duration_ar_ms FLOAT,
           allocations INTEGER,
+          queries INTEGER,
+          cached_queries INTEGER,
+          gc_duration INTEGER,
           comment TEXT,
           source_file TEXT,
           line_number INTEGER
@@ -180,9 +183,16 @@ module LogSense
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Processing by CONTROLLER as FORMAT
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Parameters: JSON
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Rendered VIEW within LAYOUT (Duration: DURATION | Allocations: ALLOCATIONS)
+        #
+        # For rails_6:
+        #
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Completed STATUS STATUS_STRING in DURATION (Views: DURATION | ActiveRecord: DURATION | Allocations: NUMBER)
         #
-        # and they appears in the order shown above: started, processing, ...
+        # For rails_7:
+        #
+        # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Completed STATUS STATUS_STRING in DURATION (Views: DURATION | ActiveRecord: DURATION (N queries, M cached) | GC: DURATION)
+        #
+        # and they appear in the order shown above: started, processing, ...
         #
         # Different requests might be interleaved, of course
         #
@@ -434,6 +444,7 @@ module LogSense
       STATUS = '(?<status>[0-9]+)'
       STATUS_IN_WORDS = '(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)'
       MSECS = '[0-9.]+'
+      INT = '[0-9]+'
 
       # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
       STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Started #{VERB} "#{URL}" for #{IP} at/o
@@ -455,7 +466,21 @@ module LogSense
       # I, [2021-10-19T08:16:34.712331 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
       # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
       # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
-      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/o
+
+      # REGEXP Fragments
+
+      # views is optional
+      # In strings I need to properly escape \ or it will be mis-interpreted
+      VIEWS = "(Views: (?<views>#{MSECS})ms \\| )?"
+      # active records has two formats (according to Rails version)
+      ACTIVE_RECORDS_BASE = "ActiveRecord: (?<arec>#{MSECS})ms"
+      ACTIVE_RECORDS_QUERIES = "( \\((?<queries>#{INT}) queries, (?<cached_queries>#{INT}) cached\\))?"
+      ACTIVE_RECORDS = "#{ACTIVE_RECORDS_BASE}#{ACTIVE_RECORDS_QUERIES}"
+      # older rails versions have ALLOCATIONS, newer have GC
+      ALLOCATIONS = " \\| Allocations: (?<alloc>#{INT})"
+      GC = " \\| GC: (?<gc_duration>#{MSECS})ms"
+
+      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \(#{VIEWS}#{ACTIVE_RECORDS}(#{ALLOCATIONS}|#{GC})?\)/o
 
       def match_and_process_completed(line)
         matchdata = (COMPLETED_REGEXP.match line)
@@ -469,7 +494,10 @@ module LogSense
             duration_total_ms: matchdata[:total],
             duration_views_ms: matchdata[:views],
             duration_ar_ms: matchdata[:arec],
-            allocations: matchdata[:alloc],
+            allocations: (matchdata[:alloc] || -1),
+            queries: (matchdata[:queries] || -1),
+            cached_queries: (matchdata[:cached_queries] || -1),
+            gc_duration: (matchdata[:gc_duration] || -1),
             comment: ""
           }
         end

data/lib/log_sense/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LogSense
-  VERSION = "2.3.0"
+  VERSION = "2.4.0"
 end

data/log_sense.gemspec

@@ -6,8 +6,18 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adolfo Villafiorita"]
   spec.email         = ["adolfo@shair.tech"]
 
-  spec.summary       = %q{Generate analytics for Rails and Apache/Nginx log file.}
-  spec.description   = %q{Generate analytics in HTML, txt, and SQLite format for Rails and Apache/Nginx log files.}
+  spec.summary       = %q{Analyze Rails and Apache/Nginx log file, generating statistics and reports.}
+  spec.description   = %q{log_sense is a command-line command which analyzes Rails and Nginx logs and
+    provides detailed data about accesses, performances, and errors.  It is a
+    one stop shop solution for analyzing your Rails application in production.
+
+    Simple to use and working offline, it does not require any modification to
+    your app (e.g., no need to change the log format, no cookies, no JavaScript
+    code, no infrastructure to setup).
+
+    We use it to monitor our applications in production, using a cron job which
+    pulls the logs and calls log_sense to generate HTML reports.
+  }
   spec.homepage      = "https://github.com/shair-tech/log_sense/"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.6.9")
@@ -26,15 +36,17 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-
-  spec.add_dependency "browser", "~> 5.3.0"
+  
+  spec.add_dependency "ostruct"
+  spec.add_dependency "csv"
+  spec.add_dependency "browser", "~> 6.0.0"
   spec.add_dependency "ipaddr", "~> 1.2.0"
   spec.add_dependency "iso_country_codes", "~> 0.7.0" 
-  spec.add_dependency "sqlite3", "~> 2.0.0"
-  spec.add_dependency "terminal-table", "~> 3.0.0"
+  spec.add_dependency "sqlite3", "~> 2.6.0"
+  spec.add_dependency "terminal-table", "~> 4.0.0"
 
-  spec.add_development_dependency "debug", "~> 1.9.0"
+  spec.add_development_dependency "debug"
   spec.add_development_dependency "rubocop"
   spec.add_development_dependency "bundle-audit"
-  spec.add_development_dependency "minitest", "~> 5.24.0"
+  spec.add_development_dependency "minitest"
 end

metadata

@@ -1,29 +1,56 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-25 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: browser
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.3.0
+        version: 6.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.3.0
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
   name: ipaddr
   requirement: !ruby/object:Gem::Requirement
@@ -58,42 +85,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -126,18 +153,24 @@ dependencies:
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.24.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.24.0
-description: Generate analytics in HTML, txt, and SQLite format for Rails and Apache/Nginx
-  log files.
+        version: '0'
+description: "log_sense is a command-line command which analyzes Rails and Nginx logs
+  and\n    provides detailed data about accesses, performances, and errors.  It is
+  a\n    one stop shop solution for analyzing your Rails application in production.\n\n
+  \   Simple to use and working offline, it does not require any modification to\n
+  \   your app (e.g., no need to change the log format, no cookies, no JavaScript\n
+  \   code, no infrastructure to setup).\n\n    We use it to monitor our applications
+  in production, using a cron job which\n    pulls the logs and calls log_sense to
+  generate HTML reports.\n  "
 email:
 - adolfo@shair.tech
 executables:
@@ -203,7 +236,6 @@ metadata:
   homepage_uri: https://github.com/shair-tech/log_sense/
   source_code_uri: https://github.com/shair-tech/log_sense/
   changelog_uri: https://github.com/shair-tech/log_sense/blob/main/CHANGELOG.org
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -218,8 +250,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
-summary: Generate analytics for Rails and Apache/Nginx log file.
+summary: Analyze Rails and Apache/Nginx log file, generating statistics and reports.
 test_files: []