log_sense 2.4.0 → 2.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01b7c715a9aac207ec39985ba5791c307e64784ae1c5c3f5f5a4e0b124a796af
-  data.tar.gz: 7f1881b1a621ea2d1b2f3f29052d4a82016d23bcb6c0d702712b249308bbf2cc
+  metadata.gz: 8f8391c34a76ec3e73970663c4636b30b357a0393b37d226cf11b28cf280e690
+  data.tar.gz: edf38e10347cee43dd31ca77bc4631da40c3dd1504949eba04806533914ee048
 SHA512:
-  metadata.gz: 0e8ccec75c78ca240320c3be45cf47779879eef0cd7033fee0e623647b29c744794e19e9f4b4f2e444e9b474b90b017ab10b60bea6661f92aaf156923aff839e
-  data.tar.gz: b70eb6964f19f0e7272b3953a4df5b124ea95c1d75ea6fd1a33f9ffe0ad04f6ddb2c7e9e177ccd19dded06a94ae159b3333fca9fadb1e85c279b87cccf78717c
+  metadata.gz: d54dd1a3014112cb2065d359e8309821321af2aa8dfd60effccf36f03735c796035be41e66f9a52f16809ea9fb3e0a179d70c7952554714b799cc74b18604ca1
+  data.tar.gz: e499045dbe77afb688b56ba01108a6d9bcbb2033c59df0c568d12c5807218f8f62579cdebc7fe35356271c51f8b86ba8aad7b72084025ec5ac6969d2b8afcf15

data/CHANGELOG.org

@@ -2,6 +2,14 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.5.1
+
+- Add number of events in SQL queries  
+
+* 2.5.0
+
+- Add query reports  
+
 * 2.4.0
 
 - Updates log_parser to match changes in log lines introduced from Rails 7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (2.3.1)
+    log_sense (2.5.1)
       browser (~> 6.0.0)
       csv
       ipaddr (~> 1.2.0)

data/lib/log_sense/formatting_util.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module LogSense
+  module FormattingUtil
+    ##
+    # Number with thousand separator
+    #
+    def self.with_thousands(number, separator: ",", comma: ".")
+      return unless number
+          
+      decimal, fraction = number.to_s.split(comma)
+      with_thousands = decimal.reverse.gsub(/(\d\d\d)/, "\\1#{separator}")
+
+      "#{with_thousands.reverse}#{comma if fraction}#{fraction}"
+    end
+  end
+end

data/lib/log_sense/rails/log_parser.rb

@@ -16,53 +16,43 @@ module LogSense
       def parse(streams, options = {})
         db = SQLite3::Database.new ":memory:"
         
+        event_table = {
+          exit_status: "TEXT",
+          started_at: "TEXT",
+          ended_at: "TEXT",
+          log_id: "TEXT",
+          ip: "TEXT",
+          unique_visitor: "TEXT",
+          url: "TEXT",
+          controller: "TEXT",
+          html_verb: "TEXT",
+          status: "INTEGER",
+          duration_total_ms: "FLOAT",
+          duration_views_ms: "FLOAT",
+          duration_ar_ms: "FLOAT",
+          allocations: "INTEGER",
+          queries: "INTEGER",
+          cached_queries: "INTEGER",
+          gc_duration: "FLOAT",
+          comment: "TEXT",
+          source_file: "TEXT",
+          line_number: "INTEGER"
+        }
+
+        table_declaration = event_table.map { |k, v| "#{k} #{v}" }.join(",")
         db.execute <<-EOS
-        CREATE TABLE IF NOT EXISTS Event(
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          exit_status TEXT,
-          started_at TEXT,
-          ended_at TEXT,
-          log_id TEXT,
-          ip TEXT,
-          unique_visitor TEXT,
-          url TEXT,
-          controller TEXT,
-          html_verb TEXT,
-          status INTEGER,
-          duration_total_ms FLOAT,
-          duration_views_ms FLOAT,
-          duration_ar_ms FLOAT,
-          allocations INTEGER,
-          queries INTEGER,
-          cached_queries INTEGER,
-          gc_duration INTEGER,
-          comment TEXT,
-          source_file TEXT,
-          line_number INTEGER
-         )
+          CREATE TABLE IF NOT EXISTS Event(
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            #{table_declaration}
+        )
         EOS
 
+        tds = event_table.keys.size
         ins = db.prepare <<-EOS
-        insert into Event(
-          exit_status,
-          started_at,
-          ended_at,
-          log_id,
-          ip,
-          unique_visitor,
-          url,
-          controller,
-          html_verb,
-          status,
-          duration_total_ms,
-          duration_views_ms,
-          duration_ar_ms,
-          allocations,
-          comment,
-          source_file,
-          line_number
+         insert into Event(
+           #{event_table.keys.join(", ")}
          )
-         values (#{Array.new(17, "?").join(", ")})
+         values (#{Array.new(tds, "?").join(", ")})
         EOS
 
         db.execute <<-EOS
@@ -254,6 +244,9 @@ module LogSense
                   event[:duration_views_ms],
                   event[:duration_ar_ms],
                   event[:allocations],
+                  event[:queries],
+                  event[:cached_queries],
+                  event[:gc_duration],
                   event[:comment],
                   filename,
                   line_number
@@ -290,6 +283,9 @@ module LogSense
                   event[:duration_views_ms],
                   event[:duration_ar_ms],
                   event[:allocations],
+                  event[:queries],
+                  event[:cached_queries],
+                  event[:gc_duration],
                   event[:comment],
                   filename,
                   line_number
@@ -494,10 +490,10 @@ module LogSense
             duration_total_ms: matchdata[:total],
             duration_views_ms: matchdata[:views],
             duration_ar_ms: matchdata[:arec],
-            allocations: (matchdata[:alloc] || -1),
-            queries: (matchdata[:queries] || -1),
-            cached_queries: (matchdata[:cached_queries] || -1),
-            gc_duration: (matchdata[:gc_duration] || -1),
+            allocations: matchdata[:alloc],
+            queries: matchdata[:queries],
+            cached_queries: matchdata[:cached_queries],
+            gc_duration: matchdata[:gc_duration],
             comment: ""
           }
         end

data/lib/log_sense/rails_aggregator.rb

@@ -5,6 +5,9 @@ module LogSense
   class RailsAggregator < Aggregator
     WORDS_SEPARATOR = ' · '
 
+    # with thousands
+    include LogSense::FormattingUtil
+
     def initialize(db, options = { limit: 900 })
       @table = "Event"
       @date_field = "started_at"
@@ -81,6 +84,30 @@ module LogSense
         }
       end
 
+      queries = @db.execute %Q(
+        SELECT SUM(DISTINCT(id)), SUM(queries), SUM(cached_queries),
+               ROUND(1.0 * SUM(cached_queries) / SUM(queries), 2)
+        FROM Event
+      )
+
+      @queries = queries.map do |row|
+        row.map do |element|
+          FormattingUtil.with_thousands(element)
+        end
+      end
+
+      @queries_by_controller = @db.execute %Q(
+        SELECT controller,
+               COUNT(DISTINCT(id)),
+               MIN(queries), MAX(queries),
+               ROUND(AVG(queries), 2),
+               SUM(queries), SUM(cached_queries),
+               ROUND(1.0 * SUM(cached_queries) / SUM(queries), 2),
+               ROUND(SUM(gc_duration), 2)
+        FROM Event
+        GROUP BY Event.controller
+      )
+
       @controller_and_methods_by_device = @db.execute %Q(
           SELECT controller as Controller,
                  method as Method,
@@ -125,7 +152,7 @@ module LogSense
                  ON event.log_id == error.log_id
                  WHERE #{filter} and exit_status == 'S:FAILED'
                  GROUP BY strftime("%Y-%m-%d", started_at)
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @fatal = @db.execute %(
           SELECT strftime("%Y-%m-%d %H:%M", started_at),
@@ -137,7 +164,7 @@ module LogSense
                  FROM Event JOIN Error
                  ON event.log_id == error.log_id
                  WHERE #{filter} and exit_status == 'S:FAILED'
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @fatal_grouped = @db.execute %(
          SELECT filename,
@@ -147,7 +174,7 @@ module LogSense
                 count(distinct(error.id))
                 FROM Error
                 GROUP BY description
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
       
       @job_plot = @db.execute %(
           SELECT strftime("%Y-%m-%d", ended_at) as Day,
@@ -156,7 +183,7 @@ module LogSense
                  FROM Job
                  WHERE #{filter}
                  GROUP BY strftime("%Y-%m-%d", ended_at)
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       # worker,
       # host,
@@ -173,7 +200,7 @@ module LogSense
                  attempt
                  FROM Job
                  WHERE #{filter}
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @job_error_grouped = @db.execute %(
          SELECT worker,
@@ -188,7 +215,7 @@ module LogSense
                 FROM Job
                 WHERE #{filter} and (exit_status == 'S:ERROR' or exit_status == 'S:FAILED')
                 GROUP BY object_id
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       instance_vars_to_hash
     end

data/lib/log_sense/rails_report_shaper.rb

@@ -46,6 +46,8 @@ module LogSense
         total_statuses(data),
         daily_statuses(data),
         performance_over_time(data, col: "small-12 cell"),
+        queries(data),
+        queries_by_controller(data),
         {
           title: "Rails Performance",
           header: %w[Controller Hits Min Avg Max],

data/lib/log_sense/report_shaper.rb

@@ -136,6 +136,30 @@ module LogSense
       }
     end
 
+    def queries(data, colors: [], col: "small-12 cell")
+      {
+        title: "Number of queries",
+        header: %w[Queries Total Cached Perc_Cached],
+        column_alignment: %i[center center center center],
+        rows: data[:queries],
+        col: col,
+      }      
+    end
+
+    def queries_by_controller(data, colors: [], col: "small-12 cell")
+      {
+        title: "Queries by Controller",
+        header: ["Controller",
+                 "Events",
+                 "Min queries", "Max queries", "Avg Queries",
+                 "Total queries", "Cached queries", "Perc",
+                 "Total GC"],
+        column_alignment: %i[left right right right right right right right],
+        rows: data[:queries_by_controller],
+        col: col,
+      }      
+    end
+
     #
     # Reports shared between rails and apache/nginx
     #

data/lib/log_sense/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LogSense
-  VERSION = "2.4.0"
+  VERSION = "2.5.1"
 end

data/lib/log_sense.rb

@@ -6,6 +6,7 @@ require "log_sense/options/checker"
 require "log_sense/apache/log_parser"
 require "log_sense/rails/log_parser"
 
+require "log_sense/formatting_util"
 require "log_sense/aggregator"
 require "log_sense/apache_aggregator"
 require "log_sense/rails_aggregator"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.1
 platform: ruby
 authors:
 - Adolfo Villafiorita
@@ -196,6 +196,7 @@ files:
 - lib/log_sense/apache_aggregator.rb
 - lib/log_sense/apache_report_shaper.rb
 - lib/log_sense/emitter.rb
+- lib/log_sense/formatting_util.rb
 - lib/log_sense/ip_locator.rb
 - lib/log_sense/options/checker.rb
 - lib/log_sense/options/parser.rb