log_sense 1.0.7 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f2fcc8402c27ec959854e3cd032e9e75693e538d3ca0879de3f4cf0b6aae9c5
-  data.tar.gz: e2530ff646c436ca88e674a698bf71876f2e62bc0b59f20ebbc4e38fdd787830
+  metadata.gz: 45cf68bf2165554a6392a9277c121952e05dd9a283fe59b881ef957f3682920e
+  data.tar.gz: 99ded91c60080ffbfda3e60f05ee4b058e215286adfccdb2a727a50d2c7d0467
 SHA512:
-  metadata.gz: 30932a09a0a0d0717c4e5caa9a579bb60a3a930dc144dc318029945e0364e71e5ddf5d5868ccf758a11d3ed5f73f5d690f482521e3475f1f16a5a041fa4abd8a
-  data.tar.gz: da8ac76f281cf9f8c67805621fe098d1480889f2fb5a55a378d1ffae3383c1913d572c28c9a68c876fb710b4e8bc3213b157c51cdf008f1505c320ef5fabedd0
+  metadata.gz: 45674016ae3c3110584e967dd60166391a07b1401186428e7e949c012bfc883686110e86afeec5450e03d34a8b7831c6bbc1d1b9393215808a2d1feafcd6422b
+  data.tar.gz: a4e9b7bf983d23ba4e8ac744621e34c5b83eee45076cf165ba9eb856d5687d47191ab048be6fed0ca4316448c84d8f80e1b7daedb518d94a90b52b7f94ee2077

data/.gitignore

@@ -6,5 +6,5 @@
 /pkg/
 /spec/reports/
 /tmp/
-
+node_modules
 *~

data/lib/log_sense/ip_locator.rb

@@ -34,6 +34,8 @@ module LogSense
     end
 
     def self.locate_ip ip, db
+      return if not ip
+
       ip_n = IPAddr.new(ip).to_i
       res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
       begin

data/lib/log_sense/options_parser.rb

@@ -22,7 +22,7 @@ module LogSense
           args[:input_file] = n
         end
 
-        opts.on("-tFORMAT", "--output-format=FORMAT", String, "Output format: html, org, txt, sqlite. Defaults to org mode") do |n|
+        opts.on("-tFORMAT", "--output-format=FORMAT", String, "Output format: html, org, txt, sqlite. See below for available formats") do |n|
           args[:output_format] = n
         end
 
@@ -74,6 +74,7 @@ module LogSense
           components = templates.map { |x| File.basename(x).split "." }.group_by { |x| x[0] }
           components.each do |k, vs|
             puts "#{k} parsing can produce the following outputs:"
+            puts "  - sqlite"
             vs.each do |v|
               puts "  - #{v[1]}"
             end

data/lib/log_sense/rails_data_cruncher.rb

@@ -15,8 +15,8 @@ module LogSense
 
       # make first and last day into dates or nil
       # TODO: bug possible value here: [[nil]], which is not empty
-      @first_day = ! first_day_s[0][0] ? nil : Date.parse(first_day_s[0][0])
-      @last_day = ! last_day_s[0][0] ? nil : Date.parse(last_day_s[0][0])
+      @first_day = first_day_s&.first&.first ? Date.parse(first_day_s[0][0]) : nil
+      @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
 
       @total_days = 0
       if @first_day and @last_day
@@ -100,6 +100,8 @@ module LogSense
 
       @performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
 
+      @fatal = db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, log_id FROM Event WHERE exit_status == 'F'"
+
       data = {}
       self.instance_variables.each do |variable|
         var_as_symbol = variable.to_s[1..-1].to_sym

data/lib/log_sense/rails_log_parser.rb

@@ -8,6 +8,7 @@ module LogSense
       db = SQLite3::Database.new ":memory:"
       db.execute 'CREATE TABLE IF NOT EXISTS Event(
          id INTEGER PRIMARY KEY AUTOINCREMENT,
+         exit_status TEXT,
          started_at TEXT,
          ended_at TEXT,
          log_id TEXT,
@@ -20,10 +21,12 @@ module LogSense
          duration_total_ms FLOAT,
          duration_views_ms FLOAT,
          duration_ar_ms FLOAT,
-         allocations INTEGER
+         allocations INTEGER,
+         comment TEXT
       )'
       
       ins = db.prepare("insert into Event(
+         exit_status,
          started_at,
          ended_at,
          log_id,
@@ -36,8 +39,10 @@ module LogSense
          duration_total_ms,
          duration_views_ms,
          duration_ar_ms,
-         allocations)
-      values (#{Array.new(13, '?').join(', ')})")
+         allocations,
+         comment
+      )
+      values (#{Array.new(15, '?').join(', ')})")
 
       # requests in the log might be interleaved.
       #
@@ -61,7 +66,7 @@ module LogSense
       
       File.readlines(filename).each do |line|
         # We discard LOG_LEVEL != 'I'
-        next if line[0] != 'I'
+        next if line[0] != 'I' and line[0] != 'F'
         
         data = self.match_and_process_start line
         if data
@@ -77,6 +82,37 @@ module LogSense
           next
         end
 
+        data = self.match_and_process_fatal line
+        if data
+          id = data[:log_id]
+          # it might as well be that the first event started before
+          # the log.  With this, we make sure we add only events whose
+          # start was logged and parsed
+          if pending[id]
+            event = data.merge (pending[id] || {})
+
+            ins.execute(
+              event[:exit_status],
+              event[:started_at],
+              event[:ended_at],
+              event[:log_id],
+              event[:ip],
+              "#{DateTime.parse(event[:started_at]).strftime("%Y-%m-%d")} #{event[:ip]}",
+              event[:url],
+              event[:controller],
+              event[:html_verb],
+              event[:status],
+              event[:duration_total_ms],
+              event[:duration_views_ms],
+              event[:duration_ar_ms],
+              event[:allocations],
+              event[:comment]
+            )
+
+            pending.delete(id)
+          end
+        end
+
         data = self.match_and_process_completed line
         if data
           id = data[:log_id]
@@ -88,6 +124,40 @@ module LogSense
             event = data.merge (pending[id] || {})
 
             ins.execute(
+              event[:exit_status],
+              event[:started_at],
+              event[:ended_at],
+              event[:log_id],
+              event[:ip],
+              "#{DateTime.parse(event[:started_at]).strftime("%Y-%m-%d")} #{event[:ip]}",
+              event[:url],
+              event[:controller],
+              event[:html_verb],
+              event[:status],
+              event[:duration_total_ms],
+              event[:duration_views_ms],
+              event[:duration_ar_ms],
+              event[:allocations],
+              event[:comment]
+            )
+
+            pending.delete(id)
+          end
+        end
+
+
+        data = self.match_and_process_completed_no_alloc line
+        if data
+          id = data[:log_id]
+
+          # it might as well be that the first event started before
+          # the log.  With this, we make sure we add only events whose
+          # start was logged and parsed
+          if pending[id]
+            event = data.merge (pending[id] || {})
+
+            ins.execute(
+              event[:exit_status],
               event[:started_at],
               event[:ended_at],
               event[:log_id],
@@ -100,12 +170,14 @@ module LogSense
               event[:duration_total_ms],
               event[:duration_views_ms],
               event[:duration_ar_ms],
-              event[:allocations]
+              event[:allocations],
+              event[:comment]
             )
 
             pending.delete(id)
           end
         end
+
       end
       
       db
@@ -141,9 +213,10 @@ module LogSense
     COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms \| Allocations: (?<alloc>[0-9]+)\)/
 
     def self.match_and_process_completed line
-      matchdata = COMPLETED_REGEXP.match line
+      matchdata = (COMPLETED_REGEXP.match line)
       if matchdata
         {
+          exit_status: "I",
           ended_at: matchdata[:timestamp],
           log_id: matchdata[:id],
           status: matchdata[:status],
@@ -151,12 +224,36 @@ module LogSense
           duration_views_ms: matchdata[:views],
           duration_ar_ms: matchdata[:arec],
           allocations: matchdata[:alloc],
+          comment: ""
+        }
+      else
+        nil
+      end
+    end
+
+    # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
+    COMPLETED_NO_ALLOC_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms\)/
+
+    def self.match_and_process_completed_no_alloc line
+      matchdata = (COMPLETED_NO_ALLOC_REGEXP.match line)
+      if matchdata
+        {
+          exit_status: "I",
+          ended_at: matchdata[:timestamp],
+          log_id: matchdata[:id],
+          status: matchdata[:status],
+          duration_total_ms: matchdata[:total],
+          duration_views_ms: matchdata[:views],
+          duration_ar_ms: matchdata[:arec],
+          allocations: -1,
+          comment: ""
         }
       else
         nil
       end
     end
 
+
     # I, [2021-10-19T08:16:34.345162 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
     PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Processing by (?<controller>[^ ]+) as/
 
@@ -172,6 +269,25 @@ module LogSense
       end
     end
 
+    # F, [2021-12-04T00:34:05.838973 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728]   
+    # F, [2021-12-04T00:34:05.839157 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728] ActionController::RoutingError (No route matches [GET] "/wp/wp-includes/wlwmanifest.xml"):
+    # F, [2021-12-04T00:34:05.839209 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728]   
+    # F, [2021-12-04T00:34:05.839269 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728] actionpack (5.2.4.4) lib/action_dispatch/middleware/debug_exceptions.rb:65:in `call'
+    FATAL_REGEXP = /F, \[#{TIMESTAMP} #[0-9]+\] FATAL -- : \[#{ID}\] (?<comment>.*)$/
+
+    def self.match_and_process_fatal line
+      matchdata = FATAL_REGEXP.match line
+      if matchdata
+        {
+          exit_status: "F",
+          log_id: matchdata[:id],
+          comment: matchdata[:comment]
+        }
+      else
+        nil
+      end
+    end
+
   end
 
 end

data/lib/log_sense/templates/_summary.html.erb

@@ -25,7 +25,7 @@
   </tr>
   <tr>
     <th class="avg-hits-per-unique-visits">Unique Visits</th>
-    <td class="avg-hits-per-unique-visits"><%= data[:total_hits] / data[:total_unique_visits] %></td>
+    <td class="avg-hits-per-unique-visits"><%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %></td>
   </tr>
   <tr>
     <th class="tx">Tx</th>

data/lib/log_sense/templates/_summary.txt.erb

@@ -4,7 +4,7 @@
                                      ["Days",            data[:total_days_in_analysis] ],
                                      ["Events",          data[:events] ],
                                      ["Unique Visits",   data[:total_unique_visits] ],
-                                     ["Avg. Events per Visit", data[:events] / data[:total_unique_visits] ]
+                                     ["Avg. Events per Visit", data[:total_unique_visits] != 0 ? data[:events] / data[:total_unique_visits] : "N/A" ]
                                    ]
  table
 %>

data/lib/log_sense/templates/rails.txt.erb

@@ -44,6 +44,12 @@ table.align_column(4, :right)
 table
 %>
 
+** Fatal Events
+
+<%= table = Terminal::Table.new headings: ['Date', 'IP', 'URL', 'Log ID'], rows: @data[:fatal]
+table
+%>
+
 ** Command Invocation
 
 <%= render "command_invocation.txt.erb", data: data %>

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "1.0.7"
+  VERSION = "1.0.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.8
 platform: ruby
 authors:
 - Adolfo Villafiorita