log_sense 2.3.1 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43c38b704af22a839fedec504755be79abf4307c1183efe65ee7323984c4d6bc
-  data.tar.gz: ed1b73c4b9ccc5e6ffcae7682e67124aa82dba1d9fde9ab625764207893a11dd
+  metadata.gz: 85b0ebee3c5879005dce7f3e32675a5f0319411eb74f138b2dfd7a4c21d26d43
+  data.tar.gz: 1aec3ca221196cfaf9061495e6767f5136a23a1f9d6f510fb43a28c056c1ca8c
 SHA512:
-  metadata.gz: 19857e10b0f5b72afdb491245101050a3348e267d6a9409416ab478e72b441ed8b559280f492f1c761753cbfd71abdafd5184f60095c53ca1bc0640a83c22f4a
-  data.tar.gz: 3f049b2b6756bfe9ef872021f6b34263736b12eb75ec2b06f086387550fbd87efe367ccc4348070cb492f5870b286c626fe6244ab19ea01f74684d301175b32b
+  metadata.gz: 27389b84f365fdaa02ece44dd70eabb130051ca81d26d17cd573325f8101d21c0dc83fd5c81915babfcd1e8d1f363ef777811f9676b2439e8f6b868013515700
+  data.tar.gz: e52b3e05fb838362c3938aaf20d1f6347a3a4c8025d40963c98eeae4fe4b6c01d9acf225353ce2c872c213b2f113af089de418bc0e424e4b151884f4fd37c03c

data/.gitignore

@@ -8,3 +8,6 @@
 /tmp/
 node_modules
 *~
+TAGS
+sample_logs
+ip_locations/dbip-country-lite-*.csv*

data/CHANGELOG.org

@@ -2,6 +2,16 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.5.0
+
+- Add query reports  
+
+* 2.4.0
+
+- Updates log_parser to match changes in log lines introduced from Rails 7
+  (or, in other words, fixes a bug which caused log_sense to miss events
+  from logs produced by Rails >= 7 applications)
+  
 * 2.3.1
 
 - Updates geoip db

data/Gemfile.lock

@@ -3,8 +3,10 @@ PATH
   specs:
     log_sense (2.3.1)
       browser (~> 6.0.0)
+      csv
       ipaddr (~> 1.2.0)
       iso_country_codes (~> 0.7.0)
+      ostruct
       sqlite3 (~> 2.6.0)
       terminal-table (~> 4.0.0)
 
@@ -18,6 +20,7 @@ GEM
     bundler-audit (0.9.2)
       bundler (>= 1.2.0, < 3)
       thor (~> 1.0)
+    csv (3.3.4)
     date (3.4.1)
     debug (1.10.0)
       irb (~> 1.10)
@@ -34,6 +37,7 @@ GEM
     lint_roller (1.1.0)
     mini_portile2 (2.8.9)
     minitest (5.25.5)
+    ostruct (0.6.1)
     parallel (1.27.0)
     parser (3.3.8.0)
       ast (~> 2.4.1)

data/README.org

@@ -4,8 +4,8 @@
 
 * Introduction
 
-LogSense generates reports and statistics from Ruby on Rails and Apache/Nginx
-log files.
+LogSense analyzes Apache and Rails log files, generating reports and statistics
+about usage and performace.
 
 Main features:
 
@@ -22,10 +22,11 @@ Main features:
   self polls and crawlers.
 - Reports can be generated in HTML, txt, ufw, and SQLite.  HTML reports are
   responsive and come with dark and light theme.
+- It works with Rails' vanilla log format (no need to format the log)
 
 LogSense is Written in Ruby, it runs from the command line, it is fast, and it
 can be installed on any system with a relatively recent version of Ruby.  We
-use it with Ruby 3.1.4 and 3.3.0.
+started using it with Ruby 3.1.4 and we are now at version 3.4.
 
 It is fast. On a ThinkPad P16, a 277M log file is parsed in 15 seconds,
 processing, that is, about 7740 events per second; a 569M log file is parsed in

data/lib/log_sense/rails/log_parser.rb

@@ -16,50 +16,43 @@ module LogSense
       def parse(streams, options = {})
         db = SQLite3::Database.new ":memory:"
         
+        event_table = {
+          exit_status: "TEXT",
+          started_at: "TEXT",
+          ended_at: "TEXT",
+          log_id: "TEXT",
+          ip: "TEXT",
+          unique_visitor: "TEXT",
+          url: "TEXT",
+          controller: "TEXT",
+          html_verb: "TEXT",
+          status: "INTEGER",
+          duration_total_ms: "FLOAT",
+          duration_views_ms: "FLOAT",
+          duration_ar_ms: "FLOAT",
+          allocations: "INTEGER",
+          queries: "INTEGER",
+          cached_queries: "INTEGER",
+          gc_duration: "FLOAT",
+          comment: "TEXT",
+          source_file: "TEXT",
+          line_number: "INTEGER"
+        }
+
+        table_declaration = event_table.map { |k, v| "#{k} #{v}" }.join(",")
         db.execute <<-EOS
-        CREATE TABLE IF NOT EXISTS Event(
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          exit_status TEXT,
-          started_at TEXT,
-          ended_at TEXT,
-          log_id TEXT,
-          ip TEXT,
-          unique_visitor TEXT,
-          url TEXT,
-          controller TEXT,
-          html_verb TEXT,
-          status INTEGER,
-          duration_total_ms FLOAT,
-          duration_views_ms FLOAT,
-          duration_ar_ms FLOAT,
-          allocations INTEGER,
-          comment TEXT,
-          source_file TEXT,
-          line_number INTEGER
-         )
+          CREATE TABLE IF NOT EXISTS Event(
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            #{table_declaration}
+        )
         EOS
 
+        tds = event_table.keys.size
         ins = db.prepare <<-EOS
-        insert into Event(
-          exit_status,
-          started_at,
-          ended_at,
-          log_id,
-          ip,
-          unique_visitor,
-          url,
-          controller,
-          html_verb,
-          status,
-          duration_total_ms,
-          duration_views_ms,
-          duration_ar_ms,
-          allocations,
-          comment,
-          source_file,
-          line_number
+         insert into Event(
+           #{event_table.keys.join(", ")}
          )
-         values (#{Array.new(17, "?").join(", ")})
+         values (#{Array.new(tds, "?").join(", ")})
         EOS
 
         db.execute <<-EOS
@@ -180,9 +173,16 @@ module LogSense
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Processing by CONTROLLER as FORMAT
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Parameters: JSON
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Rendered VIEW within LAYOUT (Duration: DURATION | Allocations: ALLOCATIONS)
+        #
+        # For rails_6:
+        #
         # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Completed STATUS STATUS_STRING in DURATION (Views: DURATION | ActiveRecord: DURATION | Allocations: NUMBER)
         #
-        # and they appears in the order shown above: started, processing, ...
+        # For rails_7:
+        #
+        # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Completed STATUS STATUS_STRING in DURATION (Views: DURATION | ActiveRecord: DURATION (N queries, M cached) | GC: DURATION)
+        #
+        # and they appear in the order shown above: started, processing, ...
         #
         # Different requests might be interleaved, of course
         #
@@ -244,6 +244,9 @@ module LogSense
                   event[:duration_views_ms],
                   event[:duration_ar_ms],
                   event[:allocations],
+                  event[:queries],
+                  event[:cached_queries],
+                  event[:gc_duration],
                   event[:comment],
                   filename,
                   line_number
@@ -280,6 +283,9 @@ module LogSense
                   event[:duration_views_ms],
                   event[:duration_ar_ms],
                   event[:allocations],
+                  event[:queries],
+                  event[:cached_queries],
+                  event[:gc_duration],
                   event[:comment],
                   filename,
                   line_number
@@ -434,6 +440,7 @@ module LogSense
       STATUS = '(?<status>[0-9]+)'
       STATUS_IN_WORDS = '(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)'
       MSECS = '[0-9.]+'
+      INT = '[0-9]+'
 
       # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
       STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Started #{VERB} "#{URL}" for #{IP} at/o
@@ -455,7 +462,21 @@ module LogSense
       # I, [2021-10-19T08:16:34.712331 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
       # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
       # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
-      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/o
+
+      # REGEXP Fragments
+
+      # views is optional
+      # In strings I need to properly escape \ or it will be mis-interpreted
+      VIEWS = "(Views: (?<views>#{MSECS})ms \\| )?"
+      # active records has two formats (according to Rails version)
+      ACTIVE_RECORDS_BASE = "ActiveRecord: (?<arec>#{MSECS})ms"
+      ACTIVE_RECORDS_QUERIES = "( \\((?<queries>#{INT}) queries, (?<cached_queries>#{INT}) cached\\))?"
+      ACTIVE_RECORDS = "#{ACTIVE_RECORDS_BASE}#{ACTIVE_RECORDS_QUERIES}"
+      # older rails versions have ALLOCATIONS, newer have GC
+      ALLOCATIONS = " \\| Allocations: (?<alloc>#{INT})"
+      GC = " \\| GC: (?<gc_duration>#{MSECS})ms"
+
+      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \(#{VIEWS}#{ACTIVE_RECORDS}(#{ALLOCATIONS}|#{GC})?\)/o
 
       def match_and_process_completed(line)
         matchdata = (COMPLETED_REGEXP.match line)
@@ -470,6 +491,9 @@ module LogSense
             duration_views_ms: matchdata[:views],
             duration_ar_ms: matchdata[:arec],
             allocations: matchdata[:alloc],
+            queries: matchdata[:queries],
+            cached_queries: matchdata[:cached_queries],
+            gc_duration: matchdata[:gc_duration],
             comment: ""
           }
         end

data/lib/log_sense/rails_aggregator.rb

@@ -81,6 +81,22 @@ module LogSense
         }
       end
 
+      @queries = @db.execute %Q(
+        SELECT SUM(queries), SUM(cached_queries),
+               ROUND(SUM(queries) / SUM(cached_queries), 2)
+        FROM Event
+      )
+
+      @queries_by_controller = @db.execute %Q(
+        SELECT controller,
+        MIN(queries), MAX(queries), ROUND(AVG(queries), 2),
+               SUM(queries), SUM(cached_queries),
+               ROUND(SUM(cached_queries) / SUM(queries), 2),
+               SUM(gc_duration)
+        FROM Event
+        GROUP BY Event.controller
+      )
+
       @controller_and_methods_by_device = @db.execute %Q(
           SELECT controller as Controller,
                  method as Method,
@@ -125,7 +141,7 @@ module LogSense
                  ON event.log_id == error.log_id
                  WHERE #{filter} and exit_status == 'S:FAILED'
                  GROUP BY strftime("%Y-%m-%d", started_at)
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @fatal = @db.execute %(
           SELECT strftime("%Y-%m-%d %H:%M", started_at),
@@ -137,7 +153,7 @@ module LogSense
                  FROM Event JOIN Error
                  ON event.log_id == error.log_id
                  WHERE #{filter} and exit_status == 'S:FAILED'
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @fatal_grouped = @db.execute %(
          SELECT filename,
@@ -147,7 +163,7 @@ module LogSense
                 count(distinct(error.id))
                 FROM Error
                 GROUP BY description
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
       
       @job_plot = @db.execute %(
           SELECT strftime("%Y-%m-%d", ended_at) as Day,
@@ -156,7 +172,7 @@ module LogSense
                  FROM Job
                  WHERE #{filter}
                  GROUP BY strftime("%Y-%m-%d", ended_at)
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       # worker,
       # host,
@@ -173,7 +189,7 @@ module LogSense
                  attempt
                  FROM Job
                  WHERE #{filter}
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @job_error_grouped = @db.execute %(
          SELECT worker,
@@ -188,7 +204,7 @@ module LogSense
                 FROM Job
                 WHERE #{filter} and (exit_status == 'S:ERROR' or exit_status == 'S:FAILED')
                 GROUP BY object_id
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       instance_vars_to_hash
     end

data/lib/log_sense/rails_report_shaper.rb

@@ -46,6 +46,8 @@ module LogSense
         total_statuses(data),
         daily_statuses(data),
         performance_over_time(data, col: "small-12 cell"),
+        queries(data),
+        queries_by_controller(data),
         {
           title: "Rails Performance",
           header: %w[Controller Hits Min Avg Max],

data/lib/log_sense/report_shaper.rb

@@ -136,6 +136,29 @@ module LogSense
       }
     end
 
+    def queries(data, colors: [], col: "small-12 cell")
+      {
+        title: "Number of queries",
+        header: %w[Queries Cached Perc_Cached],
+        column_alignment: %i[right right right],
+        rows: data[:queries],
+        col: col,
+      }      
+    end
+
+    def queries_by_controller(data, colors: [], col: "small-12 cell")
+      {
+        title: "Queries by Controller",
+        header: ["Controller",
+                 "Min queries", "Max queries", "Avg Queries",
+                 "Total queries", "Cached queries", "Perc",
+                 "Total GC"],
+        column_alignment: %i[left right right right right right right right],
+        rows: data[:queries_by_controller],
+        col: col,
+      }      
+    end
+
     #
     # Reports shared between rails and apache/nginx
     #

data/lib/log_sense/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LogSense
-  VERSION = "2.3.1"
+  VERSION = "2.5.0"
 end

data/log_sense.gemspec

@@ -6,8 +6,18 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adolfo Villafiorita"]
   spec.email         = ["adolfo@shair.tech"]
 
-  spec.summary       = %q{Generate analytics for Rails and Apache/Nginx log file.}
-  spec.description   = %q{Generate analytics in HTML, txt, and SQLite format for Rails and Apache/Nginx log files.}
+  spec.summary       = %q{Analyze Rails and Apache/Nginx log file, generating statistics and reports.}
+  spec.description   = %q{log_sense is a command-line command which analyzes Rails and Nginx logs and
+    provides detailed data about accesses, performances, and errors.  It is a
+    one stop shop solution for analyzing your Rails application in production.
+
+    Simple to use and working offline, it does not require any modification to
+    your app (e.g., no need to change the log format, no cookies, no JavaScript
+    code, no infrastructure to setup).
+
+    We use it to monitor our applications in production, using a cron job which
+    pulls the logs and calls log_sense to generate HTML reports.
+  }
   spec.homepage      = "https://github.com/shair-tech/log_sense/"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.6.9")
@@ -26,7 +36,9 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-
+  
+  spec.add_dependency "ostruct"
+  spec.add_dependency "csv"
   spec.add_dependency "browser", "~> 6.0.0"
   spec.add_dependency "ipaddr", "~> 1.2.0"
   spec.add_dependency "iso_country_codes", "~> 0.7.0" 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.5.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
@@ -9,6 +9,34 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: browser
   requirement: !ruby/object:Gem::Requirement
@@ -135,8 +163,14 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Generate analytics in HTML, txt, and SQLite format for Rails and Apache/Nginx
-  log files.
+description: "log_sense is a command-line command which analyzes Rails and Nginx logs
+  and\n    provides detailed data about accesses, performances, and errors.  It is
+  a\n    one stop shop solution for analyzing your Rails application in production.\n\n
+  \   Simple to use and working offline, it does not require any modification to\n
+  \   your app (e.g., no need to change the log format, no cookies, no JavaScript\n
+  \   code, no infrastructure to setup).\n\n    We use it to monitor our applications
+  in production, using a cron job which\n    pulls the logs and calls log_sense to
+  generate HTML reports.\n  "
 email:
 - adolfo@shair.tech
 executables:
@@ -218,5 +252,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.7
 specification_version: 4
-summary: Generate analytics for Rails and Apache/Nginx log file.
+summary: Analyze Rails and Apache/Nginx log file, generating statistics and reports.
 test_files: []