log_sense 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d20e6ee8a3f63c660de0bb9943255ed4c19b1da9e895bc269ff40310ec8f8796
-  data.tar.gz: 38338745e085de9b8ab82231dab82405f2933584d7d1dca4886b692a4460e2c0
+  metadata.gz: d96a22ce71f0c0266811faa1853981d1211b93687bfd2e3077b91189feb5742b
+  data.tar.gz: 014ad1230a6a83b379310ba2b93b16bfd956db19a8f087bd2c216b5e77211620
 SHA512:
-  metadata.gz: dbc95e9b65119822687af5a20ca6ea25958b5d427b2e7ca128ac0513c20faebe4f3cd00dcabfde7dc216def72526de67bf21b4afa87883101f96144e842ad7e1
-  data.tar.gz: dd2faf369c1f0bcf10ed151215e4521eacec4c1d46d5aa616f34158f33f03e24e515a0677c0cf9905c11359ffebeefb96fac868ed5b717358359527cedb39691
+  metadata.gz: 8fe6fb1a329b0cabae06199f13c70936c9c7aeaa6c99142a683414ac380bfc23fecb14d3588803f2f9afc24483bfb8f429f867633ce5029c3dc23df1c713dea4
+  data.tar.gz: 10fe88f045845a5573f8ba7aedde8d3475a0f390e6025746799525be685c6db11e5eb9148e9de3f8f05a7950ae9c912f2d67e454a285de08143789db374818c7

data/lib/log_sense/rails_data_cruncher.rb

@@ -92,11 +92,12 @@ module LogSense
 
       @statuses = db.execute "SELECT status, count(status) from Event where #{filter} group by status order by status"
 
+      @by_day_5xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '5' and #{filter} group by date(started_at)"
       @by_day_4xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '4' and #{filter} group by date(started_at)"
       @by_day_3xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '3' and #{filter} group by date(started_at)"
       @by_day_2xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '2' and #{filter} group by date(started_at)"
 
-      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
+      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx + @by_day_5xx).group_by { |x| x[0] }.to_a.map { |x|
         [x[0], x[1].map { |y| y[1] }].flatten
       }
 
@@ -104,8 +105,12 @@ module LogSense
 
       @performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
 
-      @fatal = db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, log_id FROM Event WHERE exit_status == 'F'"
+      @fatal = db.execute ("SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE exit_status == 'F'") || [[]]
 
+      @internal_server_error = (db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), status, ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE status is 500") || [[]]
+
+      @error = (db.execute "SELECT log_id, context, description, count(log_id) from Error GROUP BY description") || [[]]
+      
       data = {}
       self.instance_variables.each do |variable|
         var_as_symbol = variable.to_s[1..-1].to_sym

data/lib/log_sense/rails_log_parser.rb

@@ -1,4 +1,5 @@
 require 'sqlite3'
+require 'byebug'
 
 module LogSense
   module RailsLogParser
@@ -24,7 +25,7 @@ module LogSense
          allocations INTEGER,
          comment TEXT
       )'
-      
+
       ins = db.prepare("insert into Event(
          exit_status,
          started_at,
@@ -44,6 +45,22 @@ module LogSense
       )
       values (#{Array.new(15, '?').join(', ')})")
 
+      
+      db.execute 'CREATE TABLE IF NOT EXISTS Error(
+         id INTEGER PRIMARY KEY AUTOINCREMENT,
+         log_id TEXT,
+         context TEXT,
+         description TEXT
+      )'
+
+      ins_error = db.prepare("insert into Error(
+         log_id,
+         context,
+         description
+      )
+      values (?, ?, ?)")
+      
+
       # requests in the log might be interleaved.
       #
       # We use the 'pending' variable to progressively store data
@@ -65,8 +82,14 @@ module LogSense
       # Different requests might be interleaved, of course
       
       File.readlines(filename).each do |line|
-        # We discard LOG_LEVEL != 'I'
-        next if line[0] != 'I' and line[0] != 'F'
+        # I and F for completed requests, [ is for error messages
+        next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
+
+        data = self.match_and_process_error line
+        if data
+          ins_error.execute(data[:log_id], data[:context], data[:description])
+          next
+        end
         
         data = self.match_and_process_start line
         if data
@@ -145,39 +168,6 @@ module LogSense
           end
         end
 
-
-        data = self.match_and_process_completed_no_alloc line
-        if data
-          id = data[:log_id]
-
-          # it might as well be that the first event started before
-          # the log.  With this, we make sure we add only events whose
-          # start was logged and parsed
-          if pending[id]
-            event = data.merge (pending[id] || {})
-
-            ins.execute(
-              event[:exit_status],
-              event[:started_at],
-              event[:ended_at],
-              event[:log_id],
-              event[:ip],
-              "#{DateTime.parse(event[:ended_at]).strftime("%Y-%m-%d")} #{event[:ip]}",
-              event[:url],
-              event[:controller],
-              event[:html_verb],
-              event[:status],
-              event[:duration_total_ms],
-              event[:duration_views_ms],
-              event[:duration_ar_ms],
-              event[:allocations],
-              event[:comment]
-            )
-
-            pending.delete(id)
-          end
-        end
-
       end
       
       db
@@ -189,8 +179,30 @@ module LogSense
     URL = /(?<url>[^"]+)/
     IP = /(?<ip>[0-9.]+)/
     STATUS = /(?<status>[0-9]+)/
+    STATUS_IN_WORDS = /(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)/
     MSECS = /[0-9.]+/
 
+    # Error Messages
+    # [584cffcc-f1fd-4b5c-bb8b-b89621bd4921] ActionController::RoutingError (No route matches [GET] "/assets/foundation-icons.svg"):
+    # [fd8df8b5-83c9-48b5-a056-e5026e31bd5e] ActionView::Template::Error (undefined method `all_my_ancestor' for nil:NilClass):
+    # [d17ed55c-f5f1-442a-a9d6-3035ab91adf0] ActionView::Template::Error (undefined method `volunteer_for' for #<DonationsController:0x007f4864c564b8>
+    CONTEXT = /(?<context>[^ ]+Error)/
+    ERROR_REGEXP = /^\[#{ID}\] #{CONTEXT} \((?<description>.*)\):/
+
+    def self.match_and_process_error line
+      matchdata = ERROR_REGEXP.match line
+      if matchdata
+        {
+          log_id: matchdata[:id],
+          context: matchdata[:context],
+          description: matchdata[:description]
+        }
+      else
+        nil
+      end
+    end
+
+
     # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
     STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Started #{VERB} "#{URL}" for #{IP} at/
 
@@ -209,11 +221,15 @@ module LogSense
       end
     end
 
+    # TODO: Add regexps for the performance data (Views ...). We have three cases (view, active records, allocations), (views, active records), (active records, allocations)
     # I, [2021-10-19T08:16:34.712331 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
-    COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms \| Allocations: (?<alloc>[0-9]+)\)/
+    # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
+    # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
+    COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
 
     def self.match_and_process_completed line
       matchdata = (COMPLETED_REGEXP.match line)
+      # exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
       if matchdata
         {
           exit_status: "I",
@@ -231,29 +247,6 @@ module LogSense
       end
     end
 
-    # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
-    COMPLETED_NO_ALLOC_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms\)/
-
-    def self.match_and_process_completed_no_alloc line
-      matchdata = (COMPLETED_NO_ALLOC_REGEXP.match line)
-      if matchdata
-        {
-          exit_status: "I",
-          ended_at: matchdata[:timestamp],
-          log_id: matchdata[:id],
-          status: matchdata[:status],
-          duration_total_ms: matchdata[:total],
-          duration_views_ms: matchdata[:views],
-          duration_ar_ms: matchdata[:arec],
-          allocations: -1,
-          comment: ""
-        }
-      else
-        nil
-      end
-    end
-
-
     # I, [2021-10-19T08:16:34.345162 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
     PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Processing by (?<controller>[^ ]+) as/
 

data/lib/log_sense/templates/rails.html.erb

@@ -69,7 +69,7 @@
 
      .card-divider {
        padding: 0.2rem 0.4rem 0.2rem 0.4rem;
-       background: #0d0630;
+       background: #D30001;
        color: white;
      }
 
@@ -156,19 +156,10 @@
               "Log Structure",
               "Daily Distribution",
               "Time Distribution",
-              "Most Requested Pages",
-              "Most Requested Resources",
-              "404 on HTML Files",
-              "404 on other Resources",
-              "Attacks",
               "Statuses",
-              "Daily Statuses",
-              "Browsers",
-              "Platforms",
-              "Referers",
+              "Rails Performance",
+              "Fatal Events",
               "IPs",
-              "Geolocation",
-              "Streaks",
               "Command Invocation",
               "Performance"
             ].each do |item| %>
@@ -262,11 +253,24 @@
             },
             { title: "Rails Performance",
               header: ['Controller', 'Hits', 'Min', 'Avg', 'Max'],
-              rows: @data[:performance] },
+              rows: @data[:performance]
+            },
             { title: "Fatal Events",
-              header: ['Date', 'IP', 'URL', 'Log ID'], rows: @data[:fatal]
+              header: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal],
+              col: "small-12 cell"
+            },
+            { title: "Internal Server Errors",
+              header: ['Date', 'Status', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:internal_server_error],
+              col: "small-12 cell"
+            },
+            { title: "Errors",
+              header: ['Log ID', 'Context', 'Description', 'Count'], rows: @data[:error],
+              col: "small-12 cell"
+            },
+            { title: "IPs",
+              header: ["IPs", "Hits", "Country"],
+              rows: data[:ips]
             },
-            { title: "IPs", header: ["IPs", "Hits", "Country"], rows: data[:ips] },
           ]
           %>
           <div class="grid-x grid-margin-x">

data/lib/log_sense/templates/rails.txt.erb

@@ -26,14 +26,6 @@ table.align_column(2, :right)
 table
 %>
 
-** IP and Country
-
-<%=
-table = Terminal::Table.new headings: ['IP', 'Events', 'Country'], rows: @data[:ips]
-table.align_column(1, :right)
-table
-%>
-
 ** Rails Performance
 
 <%= table = Terminal::Table.new headings: ['Controller', 'Hits', 'Min', 'Avg', 'Max'], rows: @data[:performance]
@@ -46,7 +38,27 @@ table
 
 ** Fatal Events
 
-<%= table = Terminal::Table.new headings: ['Date', 'IP', 'URL', 'Log ID'], rows: @data[:fatal]
+<%= table = Terminal::Table.new headings: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal]
+table
+%>
+
+** Internal Server Errors
+
+<%= table = Terminal::Table.new headings: ['Date', 'Status', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:internal_server_error]
+table
+%>
+
+** Errors
+
+<%= table = Terminal::Table.new headings: ['Log ID', 'Context', 'Description', 'Count'], rows: @data[:error]
+table
+%>
+
+** IPs
+
+<%=
+table = Terminal::Table.new headings: ['IP', 'Hits', 'Country'], rows: @data[:ips]
+table.align_column(1, :right)
 table
 %>
 

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "1.1.2"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-12-17 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: apache_log-parser