log_sense 1.7.0 → 1.8.0

data/lib/log_sense/rails/log_parser.rb

@@ -37,7 +37,7 @@ module LogSense
           source_file TEXT,
           line_number INTEGER
          )
-      EOS
+        EOS
 
         ins = db.prepare <<-EOS
         insert into Event(
@@ -60,7 +60,7 @@ module LogSense
           line_number
          )
          values (#{Array.new(17, "?").join(", ")})
-      EOS
+        EOS
 
         db.execute <<-EOS
         CREATE TABLE IF NOT EXISTS Error(
@@ -71,18 +71,18 @@ module LogSense
          filename TEXT,
          line_number INTEGER
         )
-      EOS
+        EOS
 
         ins_error = db.prepare <<-EOS
-       insert into Error(
+        insert into Error(
          log_id,
          context,
          description,
          filename,
          line_number
-       )
-       values (?, ?, ?, ?, ?)
-      EOS
+        )
+        values (?, ?, ?, ?, ?)
+        EOS
 
         db.execute <<-EOS
         CREATE TABLE IF NOT EXISTS Render(
@@ -93,18 +93,46 @@ module LogSense
           filename TEXT,
           line_number INTEGER
         )
-      EOS
+        EOS
 
         ins_rendered = db.prepare <<-EOS
-       insert into Render(
+        insert into Render(
          partial,
          duration_ms,
          allocations,
          filename,
          line_number
-       )
-       values (?, ?, ?, ?, ?)
-      EOS
+        )
+        values (?, ?, ?, ?, ?)
+        EOS
+        
+        db.execute <<-EOS
+        CREATE TABLE IF NOT EXISTS BrowserInfo(
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          browser TEXT,
+          platform TEXT,
+          device_name TEXT,
+          controller TEXT,
+          method TEXT,
+          request_format TEXT,
+          anon_ip TEXT,
+          timestamp TEXT
+        )
+        EOS
+
+        ins_browser_info = db.prepare <<-EOS
+        insert into BrowserInfo(
+         browser,
+         platform,
+         device_name,
+         controller,
+         method,
+         request_format,
+         anon_ip,
+         timestamp
+        )
+        values (?, ?, ?, ?, ?, ?, ?, ?)
+        EOS
 
         # requests in the log might be interleaved.
         #
@@ -149,6 +177,19 @@ module LogSense
             # I and F for completed requests, [ is for error messages
             next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
 
+            data = match_and_process_browser_info line
+            if data
+              ins_browser_info.execute(data[:browser],
+                                       data[:platform],
+                                       data[:device_name],
+                                       data[:controller],
+                                       data[:method],
+                                       data[:request_format],
+                                       data[:anon_ip],
+                                       data[:timestamp])
+              next
+            end
+
             data = match_and_process_error line
             if data
               ins_error.execute(data[:log_id],
@@ -245,6 +286,7 @@ module LogSense
         db
       end
 
+      
       TIMESTAMP = /(?<timestamp>[^ ]+)/
       ID = /(?<id>[a-z0-9-]+)/
       VERB = /(?<verb>GET|POST|PATCH|PUT|DELETE)/
@@ -254,6 +296,24 @@ module LogSense
       STATUS_IN_WORDS = /(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)/
       MSECS = /[0-9.]+/
 
+      # I, [2024-07-01T02:21:34.339058 #1392909]  INFO -- : [815b3e28-8d6e-4741-8605-87654a9ff58c] BrowserInfo: "Unknown Browser","unknown_platform","Unknown","Devise::SessionsController","new","html","4db749654a0fcacbf3868f87723926e7405262f8d596e8514f4997dc80a3cd7e","2024-07-01T02:21:34+02:00"
+      BROWSER_INFO_REGEXP = /BrowserInfo: "(?<browser>.+)","(?<platform>.+)","(?<device_name>.+)","(?<controller>.+)","(?<method>.+)","(?<request_format>.+)","(?<anon_ip>.+)","(?<timestamp>.+)"/
+      def match_and_process_browser_info(line)
+        matchdata = BROWSER_INFO_REGEXP.match line
+        if matchdata
+          {
+            browser: matchdata[:browser],
+            platform: matchdata[:platform],
+            device_name: matchdata[:device_name],
+            controller: matchdata[:controller],
+            method: matchdata[:method],
+            request_format: matchdata[:request_format],
+            anon_ip: matchdata[:anon_ip],
+            timestamp: matchdata[:timestamp],
+          }
+        end
+      end
+
       # Error Messages
       # [584cffcc-f1fd-4b5c-bb8b-b89621bd4921] ActionController::RoutingError (No route matches [GET] "/assets/foundation-icons.svg"):
       # [fd8df8b5-83c9-48b5-a056-e5026e31bd5e] ActionView::Template::Error (undefined method `all_my_ancestor' for nil:NilClass):
@@ -261,7 +321,7 @@ module LogSense
       EXCEPTION = /[A-Za-z_0-9:]+(Error)?/
       ERROR_REGEXP = /^\[#{ID}\] (?<context>#{EXCEPTION}) \((?<description>(#{EXCEPTION})?.*)\):/
 
-      def match_and_process_error line
+      def match_and_process_error(line)
         matchdata = ERROR_REGEXP.match line
         if matchdata
           {
@@ -275,7 +335,7 @@ module LogSense
       # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
       STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Started #{VERB} "#{URL}" for #{IP} at/
 
-      def match_and_process_start line
+      def match_and_process_start(line)
         matchdata = STARTED_REGEXP.match line
         if matchdata
           {

data/lib/log_sense/rails_aggregator.rb

@@ -40,6 +40,62 @@ module LogSense
                  where #{filter}
                  group by controller order by controller).gsub("\n", "")
 
+      #
+      # Use the performance information to build a tree map.
+      # We then compute by device and show the treemap and the table by device
+      # together
+      #
+
+      # ["CompletedSurveysController#new", 14, "22.00", "51.57", "116.00"]
+      controller_and_methods = @performance.group_by { |element|
+        (element[0] || "#").split("#")[0]
+      }
+
+      @controller_and_methods_treemap = controller_and_methods.map do |key, values|
+        {
+          name: key,
+          value: values.map { |value| value[1] || 0.0 }.inject(&:+),
+          children: values.map { |value|
+            {
+              name: (value[0] || "#").split("#")[1],
+              value: value[1]
+            }
+          }
+        }
+      end
+
+      @controller_and_methods_by_device = @db.execute %Q(
+          SELECT controller as Controller,
+                 method as Method,
+                 request_format as Format,
+                 sum(iif(platform = 'ios', 1, 0)) as iOS,
+                 sum(iif(platform = 'android', 1, 0)) as Android,
+                 sum(iif(platform = 'mac', 1, 0)) as Mac,
+                 sum(iif(platform = 'windows', 1, 0)) as Windows,
+                 sum(iif(platform = 'linux', 1, 0)) as Linux,
+                 sum(iif(platform != 'ios' and platform != 'android' and platform != 'mac' and platform != 'windows' and platform != 'linux', 1, 0)) as Other,
+                 count(distinct(id)) as Total
+                 from BrowserInfo
+                 group by controller, method, request_format
+      )
+
+      #
+      # Browser Info data
+      #
+      @browsers = @db.execute %Q(
+          SELECT browser as Browser,
+                 count(distinct(id)) as Visits
+                 from BrowserInfo
+                 group by browser
+      )
+
+      @platforms = @db.execute %Q(
+          SELECT platform as Platform,
+                 count(distinct(id)) as Visits
+                 from BrowserInfo
+                 group by platform
+      )
+
       @fatal = @db.execute %Q(
           SELECT strftime("%Y-%m-%d %H:%M", started_at),
                  ip,
@@ -50,6 +106,14 @@ module LogSense
                  ON event.log_id == error.log_id
                  WHERE #{filter} and exit_status == 'F').gsub("\n", "") || [[]]
 
+      @fatal_plot = @db.execute %Q(
+          SELECT strftime("%Y-%m-%d", started_at) as Day,
+                 count(distinct(event.id)) as Errors
+                 FROM Event JOIN Error
+                 ON event.log_id == error.log_id
+                 WHERE #{filter} and exit_status == 'F'
+                 GROUP BY strftime("%Y-%m-%d", started_at)).gsub("\n", "") || [[]]
+
       @internal_server_error = @db.execute %Q(
          SELECT strftime("%Y-%m-%d %H:%M", started_at), status, ip, url,
                 error.description,
@@ -59,10 +123,19 @@ module LogSense
                 WHERE #{filter} and substr(status, 1, 1) == '5').gsub("\n", "") || [[]]
 
       @error = @db.execute %Q(
-         SELECT filename, log_id, context, description, count(log_id)
+         SELECT filename, log_id, description, count(log_id)
                 FROM Error
+                WHERE (description NOT LIKE '%No route matches%' and
+                       description NOT LIKE '%Couldn''t find%')
                 GROUP BY description).gsub("\n", "") || [[]]
       
+      @possible_attacks = @db.execute %Q(
+         SELECT filename, log_id, description, count(log_id)
+                FROM Error
+                WHERE (description LIKE '%No route matches%' or
+                       description LIKE '%Couldn''t find%')
+                GROUP BY description).gsub("\n", "") || [[]]
+
       instance_vars_to_hash
     end
   end