log_sense 1.4.0 → 1.4.1

data/lib/log_sense/rails_log_parser.rb

@@ -2,10 +2,8 @@ require 'sqlite3'
 
 module LogSense
   module RailsLogParser
-    def self.parse filename, options = {}
-      content = filename ? File.readlines(filename) : ARGF.readlines
-
-      db = SQLite3::Database.new ":memory:"
+    def self.parse(streams, options = {})
+      db = SQLite3::Database.new ':memory:'
       db.execute 'CREATE TABLE IF NOT EXISTS Event(
          id INTEGER PRIMARY KEY AUTOINCREMENT,
          exit_status TEXT,
@@ -17,12 +15,14 @@ module LogSense
          url TEXT,
          controller TEXT,
          html_verb TEXT,
-         status INTEGER, 
+         status INTEGER,
          duration_total_ms FLOAT,
          duration_views_ms FLOAT,
          duration_ar_ms FLOAT,
          allocations INTEGER,
-         comment TEXT
+         comment TEXT,
+         source_file TEXT,
+         line_number INTEGER
       )'
 
       ins = db.prepare("insert into Event(
@@ -35,30 +35,34 @@ module LogSense
          url,
          controller,
          html_verb,
-         status, 
+         status,
          duration_total_ms,
          duration_views_ms,
          duration_ar_ms,
          allocations,
-         comment
+         comment,
+         source_file,
+         line_number
       )
-      values (#{Array.new(15, '?').join(', ')})")
+      values (#{Array.new(17, '?').join(', ')})")
 
-      
       db.execute 'CREATE TABLE IF NOT EXISTS Error(
          id INTEGER PRIMARY KEY AUTOINCREMENT,
          log_id TEXT,
          context TEXT,
-         description TEXT
+         description TEXT,
+         filename TEXT,
+         line_number INTEGER
       )'
 
       ins_error = db.prepare("insert into Error(
          log_id,
          context,
-         description
+         description,
+         filename,
+         line_number
       )
-      values (?, ?, ?)")
-      
+      values (?, ?, ?, ?, ?)")
 
       # requests in the log might be interleaved.
       #
@@ -79,94 +83,101 @@ module LogSense
       # and they appears in the order shown above: started, processing, ...
       #
       # Different requests might be interleaved, of course
-      
-      File.readlines(filename).each do |line|
-        # I and F for completed requests, [ is for error messages
-        next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
-
-        data = self.match_and_process_error line
-        if data
-          ins_error.execute(data[:log_id], data[:context], data[:description])
-          next
-        end
-        
-        data = self.match_and_process_start line
-        if data
-          id = data[:log_id]
-          pending[id] = data.merge (pending[id] || {})
-          next
-        end
-
-        data = self.match_and_process_processing_by line
-        if data
-          id = data[:log_id]
-          pending[id] = data.merge (pending[id] || {})
-          next
-        end
-
-        data = self.match_and_process_fatal line
-        if data
-          id = data[:log_id]
-          # it might as well be that the first event started before
-          # the log.  With this, we make sure we add only events whose
-          # start was logged and parsed
-          if pending[id]
-            event = data.merge (pending[id] || {})
+      #
+      streams.each do |stream|
+        stream.readlines.each_with_index do |line, line_number|
+          filename = stream == $stdin ? "stdin" : stream.path
 
-            ins.execute(
-              event[:exit_status],
-              event[:started_at],
-              event[:ended_at],
-              event[:log_id],
-              event[:ip],
-              unique_visitor_id(event),
-              event[:url],
-              event[:controller],
-              event[:html_verb],
-              event[:status],
-              event[:duration_total_ms],
-              event[:duration_views_ms],
-              event[:duration_ar_ms],
-              event[:allocations],
-              event[:comment]
-            )
+          # I and F for completed requests, [ is for error messages
+          next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
 
-            pending.delete(id)
+          data = match_and_process_error line
+          if data
+            ins_error.execute(data[:log_id], data[:context], data[:description], filename, line_number)
+            next
+          end
+          
+          data = match_and_process_start line
+          if data
+            id = data[:log_id]
+            pending[id] = data.merge(pending[id] || {})
+            next
           end
-        end
-
-        data = self.match_and_process_completed line
-        if data
-          id = data[:log_id]
 
-          # it might as well be that the first event started before
-          # the log.  With this, we make sure we add only events whose
-          # start was logged and parsed
-          if pending[id]
-            event = data.merge (pending[id] || {})
+          data = match_and_process_processing_by line
+          if data
+            id = data[:log_id]
+            pending[id] = data.merge(pending[id] || {})
+            next
+          end
 
-            ins.execute(
-              event[:exit_status],
-              event[:started_at],
-              event[:ended_at],
-              event[:log_id],
-              event[:ip],
-              unique_visitor_id(event),
-              event[:url],
-              event[:controller],
-              event[:html_verb],
-              event[:status],
-              event[:duration_total_ms],
-              event[:duration_views_ms],
-              event[:duration_ar_ms],
-              event[:allocations],
-              event[:comment]
-            )
+          data = match_and_process_fatal line
+          if data
+            id = data[:log_id]
+            # it might as well be that the first event started before
+            # the log.  With this, we make sure we add only events whose
+            # start was logged and parsed
+            if pending[id]
+              event = data.merge(pending[id] || {})
+
+              ins.execute(
+                event[:exit_status],
+                event[:started_at],
+                event[:ended_at],
+                event[:log_id],
+                event[:ip],
+                unique_visitor_id(event),
+                event[:url],
+                event[:controller],
+                event[:html_verb],
+                event[:status],
+                event[:duration_total_ms],
+                event[:duration_views_ms],
+                event[:duration_ar_ms],
+                event[:allocations],
+                event[:comment],
+                filename,
+                line_number
+              )
+
+              pending.delete(id)
+            end
+          end
 
-            pending.delete(id)
+          data = self.match_and_process_completed line
+          if data
+            id = data[:log_id]
+
+            # it might as well be that the first event started before
+            # the log.  With this, we make sure we add only events whose
+            # start was logged and parsed
+            if pending[id]
+              event = data.merge (pending[id] || {})
+
+              ins.execute(
+                event[:exit_status],
+                event[:started_at],
+                event[:ended_at],
+                event[:log_id],
+                event[:ip],
+                unique_visitor_id(event),
+                event[:url],
+                event[:controller],
+                event[:html_verb],
+                event[:status],
+                event[:duration_total_ms],
+                event[:duration_views_ms],
+                event[:duration_ar_ms],
+                event[:allocations],
+                event[:comment],
+                filename,
+                line_number
+              )
+
+              pending.delete(id)
+            end
           end
         end
-
       end
       
       db
@@ -226,7 +237,7 @@ module LogSense
     # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
     COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
 
-    def self.match_and_process_completed line
+    def self.match_and_process_completed(line)
       matchdata = (COMPLETED_REGEXP.match line)
       # exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
       if matchdata
@@ -267,7 +278,7 @@ module LogSense
     # F, [2021-12-04T00:34:05.839269 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728] actionpack (5.2.4.4) lib/action_dispatch/middleware/debug_exceptions.rb:65:in `call'
     FATAL_REGEXP = /F, \[#{TIMESTAMP} #[0-9]+\] FATAL -- : \[#{ID}\] (?<comment>.*)$/
 
-    def self.match_and_process_fatal line
+    def self.match_and_process_fatal(line)
       matchdata = FATAL_REGEXP.match line
       if matchdata
         {
@@ -281,11 +292,8 @@ module LogSense
     end
 
     # generate a unique visitor id from an event
-    def self.unique_visitor_id event
+    def self.unique_visitor_id(event)
       "#{DateTime.parse(event[:started_at] || event[:ended_at] || "1970-01-01").strftime("%Y-%m-%d")} #{event[:ip]}"
     end
-
   end
-
 end
-

data/lib/log_sense/templates/_command_invocation.html.erb

@@ -4,10 +4,6 @@
       <th>CLI Command</th>
       <td><code><%= data[:command] %></code></td>
     </tr>
-    <tr>
-      <th>Input file</th>
-      <td><code><%= (data[:log_file] || "stdin") %></code></td>
-    </tr>
     <tr>
       <th>Ignore crawlers</th>
       <td><code><%= options[:ignore_crawlers] %></code></td></tr>

data/lib/log_sense/templates/_command_invocation.txt.erb

@@ -1,6 +1,7 @@
 <%=
-table = Terminal::Table.new rows: [ ["Command", data[:command] ],
-                                    ["Input file", data[:log_file] || "stdin" ]
-                                  ]
+table = Terminal::Table.new rows: [
+  ["Command", data[:command] ],
+]
+table.style = { border_i: "|" }
 table
 %>

data/lib/log_sense/templates/_output_table.html.erb

@@ -1,9 +1,3 @@
-<%
-def slugify string
-  string.downcase.gsub(/ +/, '-')
-end
-%>
-
 <table id="table-<%= index %>" class="table unstriped">
   <thead>
     <tr>
@@ -22,7 +16,7 @@ end
      <%= report[:datatable_options] + "," if report[:datatable_options] %>
      columns: [
        <% report[:header].each do |header| %>
-       { data: '<%= header %>', className: '<%= slugify(header) %>' },
+       { data: '<%= header %>', className: '<%= Emitter::slugify(header) %>' },
        <% end %>
      ]
    });

data/lib/log_sense/templates/_output_table.txt.erb

@@ -0,0 +1,13 @@
+<%=
+# shortens long URLs and long descriptions
+shortened = Emitter::shorten(report[:rows], report[:header], data[:width])
+
+# build and style the table
+table = Terminal::Table.new headings: report[:header], rows: shortened
+table.style = { border_i: "|" }
+columns = report[:header].size - 1
+(0..columns).map { |i| table.align_column(i, report[:column_alignment][i] || :left) }
+
+# return it
+table
+%>

data/lib/log_sense/templates/_performance.html.erb

@@ -15,7 +15,7 @@
     <%= data[:log_size] %> <span class="stats-list-label">Events</span>
   </li>
   <li class="stats-list-positive">
-    <td><%= "%.2f" % (data[:log_size] / data[:duration]) %>
+    <%= "%.2f" % (data[:log_size] / data[:duration]) %>
     <span class="stats-list-label">Parsed Events/sec</span>
   </li>
 </ul>

data/lib/log_sense/templates/_performance.txt.erb

@@ -1,9 +1,12 @@
 <%=
-table = Terminal::Table.new rows: [ ["Analysis started at", data[:started_at].to_s ],
-                                    ["Analysis ended at",   data[:ended_at].to_s ],
-                                    ["Duration",            "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] ],
-                                    ["Events",              "%9d" % data[:log_size] ],
-                                    ["Parsed events/sec",   "%.2f" % (data[:log_size] / data[:duration]) ] ]
+table = Terminal::Table.new rows: [
+  ["Analysis started at", data[:started_at].to_s ],
+  ["Analysis ended at", data[:ended_at].to_s ],
+  ["Duration", "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] ],
+  ["Events", "%9d" % data[:log_size] ],
+  ["Parsed events/sec", "%.2f" % (data[:log_size] / data[:duration]) ]
+]
+table.style = { border_i: "|" }
 table.align_column(2, :right)
 table
 %>

data/lib/log_sense/templates/_report_data.html.erb

@@ -1,11 +1,10 @@
 <script>
- /* this is used both by Vega and DataTable */
+ /* this is used both by Vega and DataTable for <%= report[:title] %>*/
  data_<%= index %> = [
    <% report[:rows].each do |row| %>
    {
      <% report[:header].each_with_index do |h, i| %>
-     <% resized_row = (row[i] || '').size > 150 ? "#{row[i][0..150]...}" : (row[i] || "") %>
-     "<%= h %>": <%= (row[i].class == Integer or row[i].class == Float) ? row[i] : "\"#{Emitter::escape_javascript(resized_row)}\"" %>,
+     "<%= h %>": "<%= Emitter::process row[i] %>",
      <% end %>
    },
    <% end %>

data/lib/log_sense/templates/_summary.html.erb

@@ -23,6 +23,6 @@
   </li>
   <li class="stats-list-negative">
     <%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %>
-    <span class="stats-list-label">Page Visited / Unique Visitor</span>
+    <span class="stats-list-label">Hits / Unique Visitor</span>
   </li>
 </ul>

data/lib/log_sense/templates/_summary.txt.erb

@@ -1,10 +1,13 @@
 <%=
- table = Terminal::Table.new rows: [ ["Input File",      data[:log_file] || "stdin" ],
-                                     ["Period Analyzed", "#{data[:first_day_in_analysis]} -- #{data[:last_day_in_analysis]}" ],
-                                     ["Days",            data[:total_days_in_analysis] ],
-                                     ["Events",          data[:events] ],
-                                     ["Unique Visits",   data[:total_unique_visits] ],
-                                     ["Avg. Events per Visit", data[:total_unique_visits] != 0 ? data[:events] / data[:total_unique_visits] : "N/A" ]
-                                   ]
- table
+table = Terminal::Table.new rows: [
+  ["From", data[:first_day_in_analysis]],
+  ["To", data[:last_day_in_analysis]],
+  ["Days", data[:total_days_in_analysis]],
+  ["Hits", data[:total_hits]],
+  ["Unique Visits", data[:total_unique_visits]],
+  ["Unique Visits / Day", data[:total_days_in_analysis] > 0 ? "%.2f" % (data[:total_unique_visits] / data[:total_days_in_analysis].to_f) : "N/A"],
+  ["Hits/Unique Visitor", data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A"]
+]
+table.style = { border_i: "|" }
+table
 %>

data/lib/log_sense/templates/apache.html.erb

@@ -220,217 +220,6 @@
             </article>
           </div>
 
-          <% @reports = [
-            { title: "Daily Distribution",
-              header: ["Day", "DOW", "Hits", "Visits", "Size"],
-              rows: data[:daily_distribution],
-              vega_spec: {
-                "layer": [
-                  {
-                    "mark": {
-                      "type": "line",
-                      "point": {
-                        "filled": false,
-                        "fill": "white"
-                      }
-                    },
-                    "encoding": {
-                      "y": {"field": "Hits", "type": "quantitative"}
-                    }
-                  },
-                  {
-                    "mark": {
-                      "type": "text",
-                      "color": "#3E5772",
-                      "align": "middle",
-                      "baseline": "top",
-                      "dx": -10,
-                      "yOffset": -15
-                    },
-                    "encoding": {
-                      "text": {"field": "Hits", "type": "quantitative"},
-                      "y": {"field": "Hits", "type": "quantitative"}
-                    }
-                  },
-
-                  {
-                    "mark": {
-                      "type": "line",
-                      "color": "#A52A2A",
-                      "point": {
-                        "color": "#A52A2A",
-                        "filled": false,
-                        "fill": "white",
-                      }
-                    },
-                    "encoding": {
-                      "y": {"field": "Visits", "type": "quantitative"}
-                    }
-                  },
-
-                  {
-                    "mark": {
-                      "type": "text",
-                      "color": "#A52A2A",
-                      "align": "middle",
-                      "baseline": "top",
-                      "dx": -10,
-                      "yOffset": -15
-                    },
-                    "encoding": {
-                      "text": {"field": "Visits", "type": "quantitative"},
-                      "y": {"field": "Visits", "type": "quantitative"}
-                    }
-                  },
-                  
-                ],
-                "encoding": {
-                  "x": {"field": "Day", "type": "temporal"},
-                }
-              }
-              
-            },
-            { title: "Time Distribution",
-              header: ["Hour", "Hits", "Visits", "Size"],
-              rows: data[:time_distribution],
-              vega_spec: {
-                "layer": [
-                  {
-                    "mark": "bar"
-                  },
-                  {
-                    "mark": {
-                      "type": "text",
-                      "align": "middle",
-                      "baseline": "top",
-                      "dx": -10,
-                      "yOffset": -15
-                    },
-                    "encoding": {
-                      "text": {"field": "Hits", "type": "quantitative"},
-                      "y": {"field": "Hits", "type": "quantitative"}
-                    }
-                  },
-                ],
-                "encoding": {
-                  "x": {"field": "Hour", "type": "nominal"},
-                  "y": {"field": "Hits", "type": "quantitative"}
-                }
-              }
-            },
-            { title: "20_ and 30_ on HTML pages",
-              header: ["Path", "Hits", "Visits", "Size", "Status"],
-              rows: data[:most_requested_pages],
-              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
-            },
-            { title: "20_ and 30_ on other resources",
-              header: ["Path", "Hits", "Visits", "Size", "Status"],
-              rows: data[:most_requested_resources],
-              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
-            },
-            { title: "40_ and 50_x on HTML pages",
-              header: ["Path", "Hits", "Visits", "Status"],
-              rows: data[:missed_pages],
-              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
-            },
-            { title: "40_ and 50_ on other resources",
-              header: ["Path", "Hits", "Visits", "Status"],
-              rows: data[:missed_resources],
-              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
-            },
-            { title: "Statuses",
-              header: ["Status", "Count"],
-              rows: data[:statuses],
-              vega_spec: {
-                "mark": "bar",
-                "encoding": {
-                  "x": {"field": "Status", "type": "nominal"},
-                  "y": {"field": "Count", "type": "quantitative"}
-                }
-              }
-            },
-            { title: "Daily Statuses",
-              header: ["Date", "S_2xx", "S_3xx", "S_4xx"],
-              rows: data[:statuses_by_day],
-              vega_spec: {
-                "transform": [ {"fold": ["S_2xx", "S_3xx", "S_4xx" ] }],
-                "mark": "bar",
-                "encoding": {
-                  "x": {
-                    "field": "Date",
-                    "type": "ordinal",
-                    "timeUnit": "day", 
-                  },
-                  "y": {
-                    "aggregate": "sum",
-                    "field": "value",
-                    "type": "quantitative"
-                  },
-                  "color": {
-                    "field": "key",
-                    "type": "nominal",
-                    "scale": {
-                      "domain": ["S_2xx", "S_3xx", "S_4xx"],
-                      "range": ["#228b22", "#ff8c00", "#a52a2a"]
-                    },
-                  }
-                }
-              }
-            },
-            { title: "Browsers",
-              header: ["Browser", "Hits", "Visits", "Size"],
-              rows: data[:browsers],
-              vega_spec: {
-                "layer": [
-                  { "mark": "bar" },
-                  {
-                    "mark": {
-                      "type": "text",
-                      "align": "middle",
-                      "baseline": "top",
-                      "dx": -10,
-                      "yOffset": -15
-                    },
-                    "encoding": {
-                      "text": {"field": "Hits", "type": "quantitative"},
-                    }
-                  },
-                ],
-                "encoding": {
-                  "x": {"field": "Browser", "type": "nominal"},
-                  "y": {"field": "Hits", "type": "quantitative"}
-                }
-              }
-            },
-            { title: "Platforms",
-              header: ["Platform", "Hits", "Visits", "Size"],
-              rows: data[:platforms],
-              vega_spec: {
-                "layer": [
-                  { "mark": "bar" },
-                  {
-                    "mark": {
-                      "type": "text",
-                      "align": "middle",
-                      "baseline": "top",
-                      "dx": -10,
-                      "yOffset": -15
-                    },
-                    "encoding": {
-                      "text": {"field": "Hits", "type": "quantitative"},
-                    }
-                  },
-                ],
-                "encoding": {
-                  "x": {"field": "Platform", "type": "nominal"},
-                  "y": {"field": "Hits", "type": "quantitative"}
-                }
-              }
-            },
-            { title: "IPs", header: ["IPs", "Hits", "Visits", "Size", "Country"], rows: data[:ips] },
-            { title: "Referers", header: ["Referers", "Hits", "Visits", "Size"], rows: data[:referers], col: "small-12 cell" },
-          ]
-          %>
           <div class="grid-x grid-margin-x">
             <% @reports.each_with_index do |report, index| %>
               <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
@@ -565,7 +354,7 @@
 
             <div class="small-12 large-6 cell">
               <article>
-                <h2 id="performance"> Performance</h2>
+                <h2 id="performance">Performance</h2>
 
                 <%=  render "performance.html.erb", data: data %>
               </article>