log_sense 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01b7c715a9aac207ec39985ba5791c307e64784ae1c5c3f5f5a4e0b124a796af
-  data.tar.gz: 7f1881b1a621ea2d1b2f3f29052d4a82016d23bcb6c0d702712b249308bbf2cc
+  metadata.gz: 85b0ebee3c5879005dce7f3e32675a5f0319411eb74f138b2dfd7a4c21d26d43
+  data.tar.gz: 1aec3ca221196cfaf9061495e6767f5136a23a1f9d6f510fb43a28c056c1ca8c
 SHA512:
-  metadata.gz: 0e8ccec75c78ca240320c3be45cf47779879eef0cd7033fee0e623647b29c744794e19e9f4b4f2e444e9b474b90b017ab10b60bea6661f92aaf156923aff839e
-  data.tar.gz: b70eb6964f19f0e7272b3953a4df5b124ea95c1d75ea6fd1a33f9ffe0ad04f6ddb2c7e9e177ccd19dded06a94ae159b3333fca9fadb1e85c279b87cccf78717c
+  metadata.gz: 27389b84f365fdaa02ece44dd70eabb130051ca81d26d17cd573325f8101d21c0dc83fd5c81915babfcd1e8d1f363ef777811f9676b2439e8f6b868013515700
+  data.tar.gz: e52b3e05fb838362c3938aaf20d1f6347a3a4c8025d40963c98eeae4fe4b6c01d9acf225353ce2c872c213b2f113af089de418bc0e424e4b151884f4fd37c03c

data/CHANGELOG.org

@@ -2,6 +2,10 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.5.0
+
+- Add query reports  
+
 * 2.4.0
 
 - Updates log_parser to match changes in log lines introduced from Rails 7

data/lib/log_sense/rails/log_parser.rb

@@ -16,53 +16,43 @@ module LogSense
       def parse(streams, options = {})
         db = SQLite3::Database.new ":memory:"
         
+        event_table = {
+          exit_status: "TEXT",
+          started_at: "TEXT",
+          ended_at: "TEXT",
+          log_id: "TEXT",
+          ip: "TEXT",
+          unique_visitor: "TEXT",
+          url: "TEXT",
+          controller: "TEXT",
+          html_verb: "TEXT",
+          status: "INTEGER",
+          duration_total_ms: "FLOAT",
+          duration_views_ms: "FLOAT",
+          duration_ar_ms: "FLOAT",
+          allocations: "INTEGER",
+          queries: "INTEGER",
+          cached_queries: "INTEGER",
+          gc_duration: "FLOAT",
+          comment: "TEXT",
+          source_file: "TEXT",
+          line_number: "INTEGER"
+        }
+
+        table_declaration = event_table.map { |k, v| "#{k} #{v}" }.join(",")
         db.execute <<-EOS
-        CREATE TABLE IF NOT EXISTS Event(
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          exit_status TEXT,
-          started_at TEXT,
-          ended_at TEXT,
-          log_id TEXT,
-          ip TEXT,
-          unique_visitor TEXT,
-          url TEXT,
-          controller TEXT,
-          html_verb TEXT,
-          status INTEGER,
-          duration_total_ms FLOAT,
-          duration_views_ms FLOAT,
-          duration_ar_ms FLOAT,
-          allocations INTEGER,
-          queries INTEGER,
-          cached_queries INTEGER,
-          gc_duration INTEGER,
-          comment TEXT,
-          source_file TEXT,
-          line_number INTEGER
-         )
+          CREATE TABLE IF NOT EXISTS Event(
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            #{table_declaration}
+        )
         EOS
 
+        tds = event_table.keys.size
         ins = db.prepare <<-EOS
-        insert into Event(
-          exit_status,
-          started_at,
-          ended_at,
-          log_id,
-          ip,
-          unique_visitor,
-          url,
-          controller,
-          html_verb,
-          status,
-          duration_total_ms,
-          duration_views_ms,
-          duration_ar_ms,
-          allocations,
-          comment,
-          source_file,
-          line_number
+         insert into Event(
+           #{event_table.keys.join(", ")}
          )
-         values (#{Array.new(17, "?").join(", ")})
+         values (#{Array.new(tds, "?").join(", ")})
         EOS
 
         db.execute <<-EOS
@@ -254,6 +244,9 @@ module LogSense
                   event[:duration_views_ms],
                   event[:duration_ar_ms],
                   event[:allocations],
+                  event[:queries],
+                  event[:cached_queries],
+                  event[:gc_duration],
                   event[:comment],
                   filename,
                   line_number
@@ -290,6 +283,9 @@ module LogSense
                   event[:duration_views_ms],
                   event[:duration_ar_ms],
                   event[:allocations],
+                  event[:queries],
+                  event[:cached_queries],
+                  event[:gc_duration],
                   event[:comment],
                   filename,
                   line_number
@@ -494,10 +490,10 @@ module LogSense
             duration_total_ms: matchdata[:total],
             duration_views_ms: matchdata[:views],
             duration_ar_ms: matchdata[:arec],
-            allocations: (matchdata[:alloc] || -1),
-            queries: (matchdata[:queries] || -1),
-            cached_queries: (matchdata[:cached_queries] || -1),
-            gc_duration: (matchdata[:gc_duration] || -1),
+            allocations: matchdata[:alloc],
+            queries: matchdata[:queries],
+            cached_queries: matchdata[:cached_queries],
+            gc_duration: matchdata[:gc_duration],
             comment: ""
           }
         end

data/lib/log_sense/rails_aggregator.rb

@@ -81,6 +81,22 @@ module LogSense
         }
       end
 
+      @queries = @db.execute %Q(
+        SELECT SUM(queries), SUM(cached_queries),
+               ROUND(SUM(queries) / SUM(cached_queries), 2)
+        FROM Event
+      )
+
+      @queries_by_controller = @db.execute %Q(
+        SELECT controller,
+        MIN(queries), MAX(queries), ROUND(AVG(queries), 2),
+               SUM(queries), SUM(cached_queries),
+               ROUND(SUM(cached_queries) / SUM(queries), 2),
+               SUM(gc_duration)
+        FROM Event
+        GROUP BY Event.controller
+      )
+
       @controller_and_methods_by_device = @db.execute %Q(
           SELECT controller as Controller,
                  method as Method,
@@ -125,7 +141,7 @@ module LogSense
                  ON event.log_id == error.log_id
                  WHERE #{filter} and exit_status == 'S:FAILED'
                  GROUP BY strftime("%Y-%m-%d", started_at)
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @fatal = @db.execute %(
           SELECT strftime("%Y-%m-%d %H:%M", started_at),
@@ -137,7 +153,7 @@ module LogSense
                  FROM Event JOIN Error
                  ON event.log_id == error.log_id
                  WHERE #{filter} and exit_status == 'S:FAILED'
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @fatal_grouped = @db.execute %(
          SELECT filename,
@@ -147,7 +163,7 @@ module LogSense
                 count(distinct(error.id))
                 FROM Error
                 GROUP BY description
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
       
       @job_plot = @db.execute %(
           SELECT strftime("%Y-%m-%d", ended_at) as Day,
@@ -156,7 +172,7 @@ module LogSense
                  FROM Job
                  WHERE #{filter}
                  GROUP BY strftime("%Y-%m-%d", ended_at)
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       # worker,
       # host,
@@ -173,7 +189,7 @@ module LogSense
                  attempt
                  FROM Job
                  WHERE #{filter}
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       @job_error_grouped = @db.execute %(
          SELECT worker,
@@ -188,7 +204,7 @@ module LogSense
                 FROM Job
                 WHERE #{filter} and (exit_status == 'S:ERROR' or exit_status == 'S:FAILED')
                 GROUP BY object_id
-      ).gsub("\n", "") || [[]]
+      ) || [[]]
 
       instance_vars_to_hash
     end

data/lib/log_sense/rails_report_shaper.rb

@@ -46,6 +46,8 @@ module LogSense
         total_statuses(data),
         daily_statuses(data),
         performance_over_time(data, col: "small-12 cell"),
+        queries(data),
+        queries_by_controller(data),
         {
           title: "Rails Performance",
           header: %w[Controller Hits Min Avg Max],

data/lib/log_sense/report_shaper.rb

@@ -136,6 +136,29 @@ module LogSense
       }
     end
 
+    def queries(data, colors: [], col: "small-12 cell")
+      {
+        title: "Number of queries",
+        header: %w[Queries Cached Perc_Cached],
+        column_alignment: %i[right right right],
+        rows: data[:queries],
+        col: col,
+      }      
+    end
+
+    def queries_by_controller(data, colors: [], col: "small-12 cell")
+      {
+        title: "Queries by Controller",
+        header: ["Controller",
+                 "Min queries", "Max queries", "Avg Queries",
+                 "Total queries", "Cached queries", "Perc",
+                 "Total GC"],
+        column_alignment: %i[left right right right right right right right],
+        rows: data[:queries_by_controller],
+        col: col,
+      }      
+    end
+
     #
     # Reports shared between rails and apache/nginx
     #

data/lib/log_sense/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LogSense
-  VERSION = "2.4.0"
+  VERSION = "2.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Adolfo Villafiorita