log_sense 1.3.5 → 1.5.0

data/lib/log_sense/options_parser.rb

@@ -7,46 +7,46 @@ module LogSense
     #
     # parse command line options
     #
-    def self.parse options
-      limit = 30
+    def self.parse(options)
+      limit = 900
       args = {} 
 
       opt_parser = OptionParser.new do |opts|
-        opts.banner = "Usage: log_sense [options] [logfile]"
+        opts.banner = 'Usage: log_sense [options] [logfile ...]'
 
-        opts.on("-tTITLE", "--title=TITLE", String, "Title to use in the report") do |n|
+        opts.on('-tTITLE', '--title=TITLE', String, 'Title to use in the report') do |n|
           args[:title] = n
         end
 
-        opts.on("-fFORMAT", "--input-format=FORMAT", String, "Input format (either rails or apache)") do |n|
+        opts.on('-fFORMAT', '--input-format=FORMAT', String, 'Input format (either rails or apache)') do |n|
           args[:input_format] = n
         end
 
-        opts.on("-iINPUT_FILE", "--input-file=INPUT_FILE", String, "Input file") do |n|
-          args[:input_file] = n
-        end
-
-        opts.on("-tFORMAT", "--output-format=FORMAT", String, "Output format: html, org, txt, sqlite. See below for available formats") do |n|
+        opts.on('-tFORMAT', '--output-format=FORMAT', String, 'Output format: html, org, txt, sqlite. See below for available formats') do |n|
           args[:output_format] = n
         end
 
-        opts.on("-oOUTPUT_FILE", "--output-file=OUTPUT_FILE", String, "Output file") do |n|
+        opts.on('-oOUTPUT_FILE', '--output-file=OUTPUT_FILE', String, 'Output file') do |n|
           args[:output_file] = n
         end
 
-        opts.on("-bDATE", "--begin=DATE", Date, "Consider entries after or on DATE") do |n|
+        opts.on('-bDATE', '--begin=DATE', Date, 'Consider entries after or on DATE') do |n|
           args[:from_date] = n
         end
 
-        opts.on("-eDATE", "--end=DATE", Date, "Consider entries before or on DATE") do |n|
+        opts.on('-eDATE', '--end=DATE', Date, 'Consider entries before or on DATE') do |n|
           args[:to_date] = n
         end
 
-        opts.on("-lN", "--limit=N", Integer, "Number of entries to show (defaults to #{limit})") do |n|
+        opts.on('-lN', '--limit=N', Integer, "Limit to the N most requested resources (defaults to #{limit})") do |n|
           args[:limit] = n
         end
 
-        opts.on("-cPOLICY", "--crawlers=POLICY", String, "Decide what to do with crawlers (applies to Apache Logs)") do |n|
+        opts.on('-wWIDTH', '--width=WIDTH', Integer, 'Maximum width of URL and description columns in text reports') do |n|
+          args[:width] = n
+        end
+
+        opts.on('-cPOLICY', '--crawlers=POLICY', String, 'Decide what to do with crawlers (applies to Apache Logs)') do |n|
           case n
           when 'only'
             args[:only_crawlers] = true
@@ -55,30 +55,34 @@ module LogSense
           end
         end
 
-        opts.on("-ns", "--no-selfpoll", "Ignore self poll entries (requests from ::1; applies to Apache Logs)") do
+        opts.on('-ns', '--no-selfpoll', 'Ignore self poll entries (requests from ::1; applies to Apache Logs)') do
           args[:no_selfpoll] = true
         end
 
-        opts.on("-v", "--version", "Prints version information") do
+        opts.on('--verbose', 'Inform about progress (prints to STDERR)') do
+          args[:verbose] = true
+        end
+
+        opts.on('-v', '--version', 'Prints version information') do
           puts "log_sense version #{LogSense::VERSION}"
-          puts "Copyright (C) 2021 Shair.Tech"
-          puts "Distributed under the terms of the MIT license"
+          puts 'Copyright (C) 2021 Shair.Tech'
+          puts 'Distributed under the terms of the MIT license'
           exit
         end
 
-        opts.on("-h", "--help", "Prints this help") do
+        opts.on('-h', '--help', 'Prints this help') do
           puts opts
-          puts ""
+          puts ''
           puts "This is version #{LogSense::VERSION}"
 
-          puts ""
-          puts "Output formats"
-          pathname = File.join(File.dirname(__FILE__), "templates", "*")
-          templates = Dir.glob(pathname).select { |x| ! File.basename(x).start_with? /_|#/ and ! File.basename(x).end_with? "~" }
-          components = templates.map { |x| File.basename(x).split "." }.group_by { |x| x[0] }
+          puts ''
+          puts 'Output formats'
+          pathname = File.join(File.dirname(__FILE__), 'templates', '*')
+          templates = Dir.glob(pathname).select { |x| !File.basename(x).start_with?(/_|#/) && !File.basename(x).end_with?('~') }
+          components = templates.map { |x| File.basename(x).split '.' }.group_by { |x| x[0] }
           components.each do |k, vs|
             puts "#{k} parsing can produce the following outputs:"
-            puts "  - sqlite"
+            puts '  - sqlite'
             vs.each do |v|
               puts "  - #{v[1]}"
             end
@@ -91,13 +95,14 @@ module LogSense
       opt_parser.parse!(options)
 
       args[:limit] ||= limit
-      args[:input_format] ||= "apache"
-      args[:output_format] ||= "html"
+      args[:input_format] ||= 'apache'
+      args[:output_format] ||= 'html'
       args[:ignore_crawlers] ||= false
       args[:only_crawlers] ||= false
       args[:no_selfpoll] ||= false
+      args[:verbose] ||= false
 
-      return args
+      args
     end
   end
 end

data/lib/log_sense/rails_data_cruncher.rb

@@ -7,7 +7,7 @@ module LogSense
     # @ variables are automatically put in the returned data
     #
 
-    def self.crunch db, options = { limit: 30 }
+    def self.crunch db, options = { limit: 900 }
       first_day_s = db.execute "SELECT started_at from Event where started_at not NULL order by started_at limit 1"
       # we could use ended_at to cover the full activity period, but I prefer started_at
       # with the meaning that the monitor event initiation
@@ -19,9 +19,10 @@ module LogSense
       @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
 
       @total_days = 0
-      if @first_day and @last_day
-        @total_days = (@last_day - @first_day).to_i
-      end
+      @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
+
+      # TODO should also look into Error
+      @source_files = db.execute "SELECT distinct(source_file) from Event"
 
       @log_size = db.execute "SELECT count(started_at) from Event"
       @log_size = @log_size[0][0]
@@ -103,6 +104,9 @@ module LogSense
 
       @ips = db.execute "SELECT ip, count(ip) from Event where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
+      @streaks = db.execute 'SELECT ip, substr(started_at, 1, 10), url from Event order by ip, started_at'
+      data = {}
+
       @performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
 
       @fatal = db.execute ("SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE exit_status == 'F'") || [[]]

data/lib/log_sense/rails_log_parser.rb

@@ -2,10 +2,8 @@ require 'sqlite3'
 
 module LogSense
   module RailsLogParser
-    def self.parse filename, options = {}
-      content = filename ? File.readlines(filename) : ARGF.readlines
-
-      db = SQLite3::Database.new ":memory:"
+    def self.parse(streams, options = {})
+      db = SQLite3::Database.new ':memory:'
       db.execute 'CREATE TABLE IF NOT EXISTS Event(
          id INTEGER PRIMARY KEY AUTOINCREMENT,
          exit_status TEXT,
@@ -17,12 +15,14 @@ module LogSense
          url TEXT,
          controller TEXT,
          html_verb TEXT,
-         status INTEGER, 
+         status INTEGER,
          duration_total_ms FLOAT,
          duration_views_ms FLOAT,
          duration_ar_ms FLOAT,
          allocations INTEGER,
-         comment TEXT
+         comment TEXT,
+         source_file TEXT,
+         line_number INTEGER
       )'
 
       ins = db.prepare("insert into Event(
@@ -35,30 +35,34 @@ module LogSense
          url,
          controller,
          html_verb,
-         status, 
+         status,
          duration_total_ms,
          duration_views_ms,
          duration_ar_ms,
          allocations,
-         comment
+         comment,
+         source_file,
+         line_number
       )
-      values (#{Array.new(15, '?').join(', ')})")
+      values (#{Array.new(17, '?').join(', ')})")
 
-      
       db.execute 'CREATE TABLE IF NOT EXISTS Error(
          id INTEGER PRIMARY KEY AUTOINCREMENT,
          log_id TEXT,
          context TEXT,
-         description TEXT
+         description TEXT,
+         filename TEXT,
+         line_number INTEGER
       )'
 
       ins_error = db.prepare("insert into Error(
          log_id,
          context,
-         description
+         description,
+         filename,
+         line_number
       )
-      values (?, ?, ?)")
-      
+      values (?, ?, ?, ?, ?)")
 
       # requests in the log might be interleaved.
       #
@@ -79,94 +83,101 @@ module LogSense
       # and they appears in the order shown above: started, processing, ...
       #
       # Different requests might be interleaved, of course
-      
-      File.readlines(filename).each do |line|
-        # I and F for completed requests, [ is for error messages
-        next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
-
-        data = self.match_and_process_error line
-        if data
-          ins_error.execute(data[:log_id], data[:context], data[:description])
-          next
-        end
-        
-        data = self.match_and_process_start line
-        if data
-          id = data[:log_id]
-          pending[id] = data.merge (pending[id] || {})
-          next
-        end
-
-        data = self.match_and_process_processing_by line
-        if data
-          id = data[:log_id]
-          pending[id] = data.merge (pending[id] || {})
-          next
-        end
-
-        data = self.match_and_process_fatal line
-        if data
-          id = data[:log_id]
-          # it might as well be that the first event started before
-          # the log.  With this, we make sure we add only events whose
-          # start was logged and parsed
-          if pending[id]
-            event = data.merge (pending[id] || {})
+      #
+      streams.each do |stream|
+        stream.readlines.each_with_index do |line, line_number|
+          filename = stream == $stdin ? "stdin" : stream.path
 
-            ins.execute(
-              event[:exit_status],
-              event[:started_at],
-              event[:ended_at],
-              event[:log_id],
-              event[:ip],
-              unique_visitor_id(event),
-              event[:url],
-              event[:controller],
-              event[:html_verb],
-              event[:status],
-              event[:duration_total_ms],
-              event[:duration_views_ms],
-              event[:duration_ar_ms],
-              event[:allocations],
-              event[:comment]
-            )
+          # I and F for completed requests, [ is for error messages
+          next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
 
-            pending.delete(id)
+          data = match_and_process_error line
+          if data
+            ins_error.execute(data[:log_id], data[:context], data[:description], filename, line_number)
+            next
+          end
+          
+          data = match_and_process_start line
+          if data
+            id = data[:log_id]
+            pending[id] = data.merge(pending[id] || {})
+            next
           end
-        end
-
-        data = self.match_and_process_completed line
-        if data
-          id = data[:log_id]
 
-          # it might as well be that the first event started before
-          # the log.  With this, we make sure we add only events whose
-          # start was logged and parsed
-          if pending[id]
-            event = data.merge (pending[id] || {})
+          data = match_and_process_processing_by line
+          if data
+            id = data[:log_id]
+            pending[id] = data.merge(pending[id] || {})
+            next
+          end
 
-            ins.execute(
-              event[:exit_status],
-              event[:started_at],
-              event[:ended_at],
-              event[:log_id],
-              event[:ip],
-              unique_visitor_id(event),
-              event[:url],
-              event[:controller],
-              event[:html_verb],
-              event[:status],
-              event[:duration_total_ms],
-              event[:duration_views_ms],
-              event[:duration_ar_ms],
-              event[:allocations],
-              event[:comment]
-            )
+          data = match_and_process_fatal line
+          if data
+            id = data[:log_id]
+            # it might as well be that the first event started before
+            # the log.  With this, we make sure we add only events whose
+            # start was logged and parsed
+            if pending[id]
+              event = data.merge(pending[id] || {})
+
+              ins.execute(
+                event[:exit_status],
+                event[:started_at],
+                event[:ended_at],
+                event[:log_id],
+                event[:ip],
+                unique_visitor_id(event),
+                event[:url],
+                event[:controller],
+                event[:html_verb],
+                event[:status],
+                event[:duration_total_ms],
+                event[:duration_views_ms],
+                event[:duration_ar_ms],
+                event[:allocations],
+                event[:comment],
+                filename,
+                line_number
+              )
+
+              pending.delete(id)
+            end
+          end
 
-            pending.delete(id)
+          data = self.match_and_process_completed line
+          if data
+            id = data[:log_id]
+
+            # it might as well be that the first event started before
+            # the log.  With this, we make sure we add only events whose
+            # start was logged and parsed
+            if pending[id]
+              event = data.merge (pending[id] || {})
+
+              ins.execute(
+                event[:exit_status],
+                event[:started_at],
+                event[:ended_at],
+                event[:log_id],
+                event[:ip],
+                unique_visitor_id(event),
+                event[:url],
+                event[:controller],
+                event[:html_verb],
+                event[:status],
+                event[:duration_total_ms],
+                event[:duration_views_ms],
+                event[:duration_ar_ms],
+                event[:allocations],
+                event[:comment],
+                filename,
+                line_number
+              )
+
+              pending.delete(id)
+            end
           end
         end
-
       end
       
       db
@@ -226,7 +237,7 @@ module LogSense
     # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
     COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
 
-    def self.match_and_process_completed line
+    def self.match_and_process_completed(line)
       matchdata = (COMPLETED_REGEXP.match line)
       # exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
       if matchdata
@@ -267,7 +278,7 @@ module LogSense
     # F, [2021-12-04T00:34:05.839269 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728] actionpack (5.2.4.4) lib/action_dispatch/middleware/debug_exceptions.rb:65:in `call'
     FATAL_REGEXP = /F, \[#{TIMESTAMP} #[0-9]+\] FATAL -- : \[#{ID}\] (?<comment>.*)$/
 
-    def self.match_and_process_fatal line
+    def self.match_and_process_fatal(line)
       matchdata = FATAL_REGEXP.match line
       if matchdata
         {
@@ -281,11 +292,8 @@ module LogSense
     end
 
     # generate a unique visitor id from an event
-    def self.unique_visitor_id event
+    def self.unique_visitor_id(event)
       "#{DateTime.parse(event[:started_at] || event[:ended_at] || "1970-01-01").strftime("%Y-%m-%d")} #{event[:ip]}"
     end
-
   end
-
 end
-

data/lib/log_sense/templates/_command_invocation.html.erb

@@ -4,10 +4,6 @@
       <th>CLI Command</th>
       <td><code><%= data[:command] %></code></td>
     </tr>
-    <tr>
-      <th>Input file</th>
-      <td><code><%= (data[:log_file] || "stdin") %></code></td>
-    </tr>
     <tr>
       <th>Ignore crawlers</th>
       <td><code><%= options[:ignore_crawlers] %></code></td></tr>

data/lib/log_sense/templates/_command_invocation.txt.erb

@@ -1,6 +1,7 @@
 <%=
-table = Terminal::Table.new rows: [ ["Command", data[:command] ],
-                                    ["Input file", data[:log_file] || "stdin" ]
-                                  ]
+table = Terminal::Table.new rows: [
+  ["Command", data[:command] ],
+]
+table.style = { border_i: "|" }
 table
 %>

data/lib/log_sense/templates/_navigation.html.erb

@@ -0,0 +1,21 @@
+<nav>
+  <h2>Navigation</h2>
+  <ul class="no-bullet">
+    <% (["Summary", "Log Structure"] +
+        menus +
+        ["Command Invocation", "Performance"]).each do |item| %>
+      <li class="nav-item">
+        <a href="#<%= Emitter::slugify item %>" data-close>
+          <%= item %>
+        </a>
+      </li>
+    <% end %>
+  </ul>
+
+  <p>
+    Generated by
+    <a href="https://github.com/avillafiorita/log_sense">LogSense</a> <br />
+    on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
+    <a href='https://db-ip.com'>IP Geolocation by DB-IP</a>
+  </p>
+</nav>

data/lib/log_sense/templates/_output_table.html.erb

@@ -1,9 +1,3 @@
-<%
-def slugify string
-  string.downcase.gsub(/ +/, '-')
-end
-%>
-
 <table id="table-<%= index %>" class="table unstriped">
   <thead>
     <tr>
@@ -19,9 +13,10 @@ end
  $(document).ready(function(){
    $('#table-<%= index %>').dataTable({
      data: data_<%= index %>,
+     <%= report[:datatable_options] + "," if report[:datatable_options] %>
      columns: [
        <% report[:header].each do |header| %>
-       { data: '<%= header %>', className: '<%= slugify(header) %>' },
+       { data: '<%= header %>', className: '<%= Emitter::slugify(header) %>' },
        <% end %>
      ]
    });

data/lib/log_sense/templates/_output_table.txt.erb

@@ -0,0 +1,14 @@
+<%=
+# shortens long URLs and long descriptions
+shortened = Emitter::shorten(report[:rows], report[:header], data[:width])
+
+# build and style the table
+table = Terminal::Table.new headings: report[:header], rows: shortened
+table.style = { border_i: "|" }
+columns = report[:header].size - 1
+(0..columns).map do |i|
+  table.align_column(i, report[:column_alignment][i] || :left)
+end
+# return it
+table
+%>

data/lib/log_sense/templates/_performance.html.erb

@@ -15,7 +15,7 @@
     <%= data[:log_size] %> <span class="stats-list-label">Events</span>
   </li>
   <li class="stats-list-positive">
-    <td><%= "%.2f" % (data[:log_size] / data[:duration]) %>
+    <%= "%.2f" % (data[:log_size] / data[:duration]) %>
     <span class="stats-list-label">Parsed Events/sec</span>
   </li>
 </ul>

data/lib/log_sense/templates/_performance.txt.erb

@@ -1,9 +1,12 @@
 <%=
-table = Terminal::Table.new rows: [ ["Analysis started at", data[:started_at].to_s ],
-                                    ["Analysis ended at",   data[:ended_at].to_s ],
-                                    ["Duration",            "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] ],
-                                    ["Events",              "%9d" % data[:log_size] ],
-                                    ["Parsed events/sec",   "%.2f" % (data[:log_size] / data[:duration]) ] ]
+table = Terminal::Table.new rows: [
+  ["Analysis started at", data[:started_at].to_s ],
+  ["Analysis ended at", data[:ended_at].to_s ],
+  ["Duration", "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] ],
+  ["Events", "%9d" % data[:log_size] ],
+  ["Parsed events/sec", "%.2f" % (data[:log_size] / data[:duration]) ]
+]
+table.style = { border_i: "|" }
 table.align_column(2, :right)
 table
 %>

data/lib/log_sense/templates/_report_data.html.erb

@@ -1,10 +1,10 @@
 <script>
- /* this is used both by Vega and DataTable */
+ /* this is used both by Vega and DataTable for <%= report[:title] %>*/
  data_<%= index %> = [
    <% report[:rows].each do |row| %>
    {
      <% report[:header].each_with_index do |h, i| %>
-     "<%= h %>": <%= (row[i].class == Integer or row[i].class == Float) ? row[i] : "\"#{Emitter::escape_javascript(row[i] || '')}\"" %>,
+     "<%= h %>": "<%= Emitter::process row[i] %>",
      <% end %>
    },
    <% end %>

data/lib/log_sense/templates/_summary.html.erb

@@ -17,7 +17,12 @@
     <%= data[:total_unique_visits] %> <span class="stats-list-label">Unique Visits</span>
   </li>
   <li class="stats-list-negative">
-    <%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %>
+    <% days = data[:last_day_in_analysis] - data[:first_day_in_analysis] %>
+    <%= days > 0 ? "%d" % (data[:total_unique_visits] / days) : "N/A" %>
     <span class="stats-list-label">Unique Visits / Day</span>
   </li>
+  <li class="stats-list-negative">
+    <%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %>
+    <span class="stats-list-label">Hits / Unique Visitor</span>
+  </li>
 </ul>

data/lib/log_sense/templates/_summary.txt.erb

@@ -1,10 +1,13 @@
 <%=
- table = Terminal::Table.new rows: [ ["Input File",      data[:log_file] || "stdin" ],
-                                     ["Period Analyzed", "#{data[:first_day_in_analysis]} -- #{data[:last_day_in_analysis]}" ],
-                                     ["Days",            data[:total_days_in_analysis] ],
-                                     ["Events",          data[:events] ],
-                                     ["Unique Visits",   data[:total_unique_visits] ],
-                                     ["Avg. Events per Visit", data[:total_unique_visits] != 0 ? data[:events] / data[:total_unique_visits] : "N/A" ]
-                                   ]
- table
+table = Terminal::Table.new rows: [
+  ["From", data[:first_day_in_analysis]],
+  ["To", data[:last_day_in_analysis]],
+  ["Days", data[:total_days_in_analysis]],
+  ["Hits", data[:total_hits]],
+  ["Unique Visits", data[:total_unique_visits]],
+  ["Unique Visits / Day", data[:total_days_in_analysis] > 0 ? "%d" % (data[:total_unique_visits] / data[:total_days_in_analysis]) : "N/A"],
+  ["Hits/Unique Visitor", data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A"]
+]
+table.style = { border_i: "|" }
+table
 %>

data/lib/log_sense/templates/_warning.txt.erb

@@ -0,0 +1 @@
+>>>> URLs IN THIS FILE ARE NOT SANITIZED AND MIGHT BE UNSAFE TO OPEN <<<<