log_sense 1.5.2 → 1.6.0

data/lib/log_sense/rails_report_shaper.rb

@@ -0,0 +1,183 @@
+module LogSense
+  class RailsReportShaper < ReportShaper
+    def shape(data)
+      [
+        {
+          title: "Daily Distribution",
+          header: %w[Day DOW Hits],
+          column_alignment: %i[left left right],
+          rows: data[:daily_distribution],
+          vega_spec: {
+            "encoding": {
+                          "x": {"field": "Day", "type": "temporal"},
+                         "y": {"field": "Hits", "type": "quantitative"}
+                        },
+                      "layer": [
+                                 {
+                                   "mark": {
+                                             "type": "line",
+                                            "point": {
+                                                       "filled": false,
+                                                      "fill": "white"
+                                                     }
+                                           }
+                                 },
+                                 {
+                                   "mark": {
+                                             "type": "text",
+                                            "align": "left",
+                                            "baseline": "middle",
+                                            "dx": 5
+                                           },
+                                  "encoding": {
+                                                "text": {"field": "Hits", "type": "quantitative"}
+                                              }
+                                 }
+                               ]
+          }
+        },
+        {
+          title: "Time Distribution",
+          header: %w[Hour Hits],
+          column_alignment: %i[left right],
+          rows: data[:time_distribution],
+          vega_spec: {
+            "layer": [
+                       {
+                         "mark": "bar",
+                       },
+                       {
+                         "mark": {
+                                   "type": "text",
+                                  "align": "middle",
+                                  "baseline": "top",
+                                  "dx": -10,
+                                  "yOffset": -15
+                                 },
+                        "encoding": {
+                                      "text": {"field": "Hits", "type": "quantitative"}
+                                    }
+                       }
+                     ],
+                      "encoding": {
+                                    "x": {"field": "Hour", "type": "nominal"},
+                                   "y": {"field": "Hits", "type": "quantitative"}
+                                  }
+          }
+        },
+        {
+          title: "Statuses",
+          header: %w[Status Count],
+          column_alignment: %i[left right],
+          rows: data[:statuses],
+          vega_spec: {
+            "layer": [
+                       {
+                         "mark": "bar"
+                       },
+                       {
+                         "mark": {
+                                   "type": "text",
+                                  "align": "left",
+                                  "baseline": "top",
+                                  "dx": -10,
+                                  "yOffset": -20
+                                 },
+                        "encoding": {
+                                      "text": {"field": "Count", "type": "quantitative"}
+                                    }
+                       }
+                     ],
+                      "encoding": {
+                                    "x": {"field": "Status", "type": "nominal"},
+                                   "y": {"field": "Count", "type": "quantitative"}
+                                  }
+          }
+        },
+        {
+          title: "Rails Performance",
+          header: %w[Controller Hits Min Avg Max],
+          column_alignment: %i[left right right right right],
+          rows: data[:performance],
+          vega_spec: {
+            "layer": [
+                       {
+                         "mark": { "type": "point",
+                                   "name": "data_points"
+                                 }
+                       },
+                       {
+                         "mark": { "name": "label",
+                                   "type": "text",
+                                   "align": "left",
+                                   "baseline": "middle",
+                                   "dx": 5,
+                                   "yOffset": 0
+                                 },
+                        "encoding": { "text": {"field": "Controller"},
+                                      "fontSize": {"value": 8}
+                                    },
+                       },
+                     ],
+                      "encoding": { "x": { "field": "Avg",
+                                           "type": "quantitative"
+                                         },
+                                    "y": { "field": "Hits",
+                                           "type": "quantitative"
+                                         }
+                                  },
+          }
+        },
+        {
+          title: "Fatal Events",
+          header: %w[Date IP URL Description Log ID],
+          column_alignment: %i[left left left left left],
+          rows: data[:fatal],
+          col: "small-12 cell"
+        },
+        {
+          title: "Internal Server Errors",
+          header: %w[Date Status IP URL Description Log ID],
+          column_alignment: %i[left left left left left left],
+          rows: data[:internal_server_error],
+          col: "small-12 cell"
+        },
+        {
+          title: "Errors",
+          header: %w[Log ID Context Description Count],
+          column_alignment: %i[left left left left],
+          rows: data[:error],
+          col: "small-12 cell"
+        },
+        {
+          title: "IPs",
+          header: %w[IPs Hits Country],
+          column_alignment: %i[left right left],
+          rows: data[:ips]
+        },
+        {
+          title: "Countries",
+          header: ["Country", "Hits", "IPs", "IP List"],
+          column_alignment: %i[left right left],
+          rows: countries_table(data[:countries])
+        },
+        ip_per_hour_report_spec(ips_per_hour(data[:ips_per_hour])),
+        session_report_spec(ips_detailed(data[:ips_per_day_detailed]))
+      ]
+    end
+
+    private
+
+    # { country => [[ip, visit, country], ...]
+    def countries_table(data)
+      data&.map { |k, v|
+        [
+          k,
+          v.map { |x| x[1] }.inject(&:+),
+          v.map { |x| x[0] }.uniq.size,
+          v.map { |x| x[0] }.join(WORDS_SEPARATOR)
+        ]
+      }&.sort { |x, y| x[0] <=> y[0] }
+    end
+  end
+end

data/lib/log_sense/report_shaper.rb

@@ -0,0 +1,105 @@
+module LogSense
+  class ReportShaper
+    WORDS_SEPARATOR = ' · '
+
+    # return { [ip,day] => { [ hits, list of urls ] } }
+    def ips_detailed(ips_per_day)
+      hash = pivot(ips_per_day, 0, 1, lambda { |array| array.map { |x| x[2] } })
+      array = []
+      hash.keys.map do |ip|
+        array += hash[ip].keys.map do |date|
+          [
+            ip,
+            hash[ip].keys.size,
+            date,
+            hash[ip][date].size,
+            hash[ip][date].uniq.size,
+            hash[ip][date].uniq.size < 100 ? hash[ip][date].uniq.join(WORDS_SEPARATOR) : "[too many]"
+          ]
+        end
+      end
+      array
+    end
+
+    def ips_per_hour(ips_per_hour)
+      hash = pivot(ips_per_hour, 0, 1)
+      hash.keys.map do |ip|
+        [
+          ip,
+          (0..23).map { |hour| hash[ip][hour.to_s]&.to_i }
+        ].flatten
+      end
+    end
+
+    # Shape an array of arrays into a hash of hashes (row, cols)
+    # Params:
+    # +input_data+:: array of array
+    # +row_key+::    index of the inner array or lambda, used to group data
+    #                by rows
+    # +col_key+::    index of the inner array or lambda, will be used
+    #                to group cols
+    # +cell+::       lambda to get the value of the cells to put in the hashes.
+    #                Takes and array as input (that is, an array of all the
+    #                elements resulting from grouping)
+    #
+    # Example
+    #
+    #  input_array: [[IP, HOUR, VISITS], [...]],
+    #  row_key: 0
+    #  col_key: 1
+    #  cell: lambda { |x| x[2] }
+    #
+    # Will output:
+    #
+    #  { IP => { HOUR => VISITS, HOUR => VISITS }, ... }
+    #
+    def pivot(input_data,
+              row_key,
+              col_key,
+              cell_maker = lambda { |x| x[0].last })
+      # here we build:
+      # "95.108.213.66"=>{12=>1, 18=>2, 19=>1, 20=>1, 5=>3, 6=>2, 7=>2, 3=>1},
+      #
+      # first transform: IP => [ HOUR => [[IP,HOUR,T], [IP,HOUR,T]] ]
+      # second transform: [IP,HOUR,T] -> T
+      input_data.group_by { |entry|
+        if row_key.class == Integer
+          entry[row_key]
+        else
+          row_key.call(entry)
+        end
+      }.transform_values { |rows|
+        rows.group_by { |cols|
+          if col_key.class == Integer
+            cols[col_key]
+          else
+            col_key.call(cols)
+          end
+        }.transform_values { |array|
+          cell_maker.call(array)
+        }
+      }
+    end
+
+    def session_report_spec(data)
+      {
+        title: "Sessions",
+        report: :html,
+        header: ["IP", "Days", "Date", "Visits", "Distinct URL", "URL List"],
+        column_alignment: %i[left left right right right right],
+        rows: data,
+        col: "small-12 cell"
+      }
+    end
+
+    def ip_per_hour_report_spec(data)
+      {
+        title: "IP per hour",
+        header: ["IP"] + (0..23).map { |hour| hour.to_s },
+        column_alignment: [:left] + [:right] * 24,
+        rows: data,
+        col: "small-12 cell"
+      }
+    end
+  end
+end

data/lib/log_sense/templates/_cdn_links.html.erb

@@ -0,0 +1,11 @@
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Fira+Sans:wght@300;700&display=swap" rel="stylesheet">
+
+<% LogSense::Emitter::CDN_CSS.each do |link| %>
+  <link rel="stylesheet" type="text/css" href="<%= link %>">
+<% end %>
+
+<% LogSense::Emitter::CDN_JS.each do |link| %>
+  <script type="text/javascript" src="<%= link %>"></script>
+<% end %>

data/lib/log_sense/templates/_command_invocation.html.erb

@@ -19,6 +19,10 @@
       <th>No selfpoll</th>
       <td><code><%= options[:no_selfpoll] %></code></td>
     </tr>
+    <tr>
+      <th>Geolocation</th>
+      <td><code><%= options[:geolocation] %></code></td>
+    </tr>
     <tr>
       <th>Filter by date</th>
       <td>

data/lib/log_sense/templates/_log_structure.html.erb

@@ -11,17 +11,23 @@
     <%= data[:total_days] %>
     <span class="stats-list-label">Days in Log</span>
   </li>
+  <% if data[:log_size] %>
   <li class="stats-list-negative">
     <%= data[:log_size] %>
     <span class="stats-list-label">Total Entries</span>
   </li>
+  <% end %>
+  <% if data[:selfpolls_size] %>
   <li class="stats-list-negative">
     <%= data[:selfpolls_size] %>
     <span class="stats-list-label">Self Polls</span>
   </li>
-  <li class="stats-list-negative">
+  <% end %>
+  <% if data[:crawlers] %>
+    <li class="stats-list-negative">
     <td><%= data[:crawlers_size] %></td>
     <span class="stats-list-label">Crawlers</span>
+  <% end %>
   </li>
 </ul>
 

data/lib/log_sense/templates/_output_table.html.erb

@@ -54,7 +54,7 @@
          data: 'IP',
          className: '<%= Emitter::slugify("IP") %>',
          render: function(data, type, row) {
-           // If display or filter data is requested, format the date
+           // If display or filter data is requested, format the data
            if ( type === 'display' || type === 'filter' ) {
              return "<a target=\"_blank\" href=\"https://db-ip.com/" + data + "\">" + data + "</a>";
            }
@@ -63,7 +63,11 @@
          }
        },
        <% else -%>
-       { data: '<%= header %>', className: '<%= Emitter::slugify(header) %>' },
+       {
+         data: '<%= header %>',
+         className: '<%= Emitter::slugify(header) %>',
+         render: DataTable.render.text()
+       },
        <% end -%>
        <% end -%>
      ]

data/lib/log_sense/templates/_rails.css.erb

@@ -0,0 +1,7 @@
+#offCanvas {
+    background: #d30001 !important;
+}
+h2 {
+    background: #d30001 !important;
+}
+    

data/lib/log_sense/templates/_summary.html.erb

@@ -11,12 +11,14 @@
     <%= data[:total_days_in_analysis] %>
     <span class="stats-list-label">Days</span>
   </li>
-  <li class="stats-list-positive">
-    <%= data[:total_size] %>
-    <span class="stats-list-label">Data Transferred</span>
-  </li>
+  <% if data[:total_size] %>
+    <li class="stats-list-positive">
+      <%= data[:total_size] %>
+      <span class="stats-list-label">Data Transferred</span>
+    </li>
+  <% end %>
   <li class="stats-list-negative">
-    <%= data[:total_hits] %>
+    <%= data[:events] %>
     <span class="stats-list-label">Hits</span>
   </li>
   <li class="stats-list-negative">
@@ -24,12 +26,12 @@
     <span class="stats-list-label">Unique Visits</span>
   </li>
   <li class="stats-list-negative">
-    <% days = data[:last_day_in_analysis] - data[:first_day_in_analysis] %>
+    <% days = data[:total_days_in_analysis] %>
     <%= days > 0 ? "%d" % (data[:total_unique_visits] / days) : "N/A" %>
     <span class="stats-list-label">Unique Visits / Day</span>
   </li>
   <li class="stats-list-negative">
-    <%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %>
+    <%= data[:total_unique_visits] != 0 ? data[:events] / data[:total_unique_visits] : "N/A" %>
     <span class="stats-list-label">Hits / Unique Visitor</span>
   </li>
 </ul>

data/lib/log_sense/templates/_summary.txt.erb

@@ -3,10 +3,10 @@ table = Terminal::Table.new rows: [
   ["From", data[:first_day_in_analysis]],
   ["To", data[:last_day_in_analysis]],
   ["Days", data[:total_days_in_analysis]],
-  ["Hits", data[:total_hits]],
+  ["Hits", data[:events]],
   ["Unique Visits", data[:total_unique_visits]],
   ["Unique Visits / Day", data[:total_days_in_analysis] > 0 ? "%d" % (data[:total_unique_visits] / data[:total_days_in_analysis]) : "N/A"],
-  ["Hits/Unique Visitor", data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A"]
+  ["Hits/Unique Visitor", data[:total_unique_visits] != 0 ? data[:events] / data[:total_unique_visits] : "N/A"]
 ]
 table.style = { border_i: "|" }
 table

data/lib/log_sense/templates/{rails.html.erb → report_html.erb}

@@ -1,42 +1,22 @@
 <!doctype html>
 <html class="no-js" lang="en">
   <head>
-    <title>
-      <%= options[:title] || "Log Sense: #{data[:filenames].empty? ? "stdin" : data[:filenames].join(", ")}" %>
-    </title>
+    <title><%= @report_title %></title>
 
     <meta charset="utf-8" />
     <meta http-equiv="x-ua-compatible" content="ie=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 
     <meta name="author" content="Shair.Tech">
-    <meta name="description" content="Analysis of <%= data[:filenames].join(', ') %>">
+    <meta name="description"
+          content="Analysis of <%= @data[:filenames].join(', ') %>">
 
-    <link rel="preconnect" href="https://fonts.googleapis.com">
-    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-    <link href="https://fonts.googleapis.com/css2?family=Fira+Sans:wght@300;700&display=swap" rel="stylesheet">
-
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/foundicons/3.0.0/foundation-icons.min.css">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/css/foundation.min.css">
-    <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.css"/>
-
-    <script type="text/javascript" src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
-    <script type="text/javascript" src="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.js"></script>
-
-    <script src="https://cdn.jsdelivr.net/npm/vega@5.21.0"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega-lite@5.2.0"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega-embed@6.20.2"></script>
+    <%= render "cdn_links.html.erb" %>
 
     <style>
      <%= render "stylesheet.css" %>
-    </style>
-    <style>
-     #offCanvas {
-       background: #d30001 !important;
-     }
-     h2 {
-       background: #d30001 !important;
-     }
+
+     <%= render @format_specific_css %>
     </style>
   </head>
 
@@ -44,7 +24,7 @@
     <div class="off-canvas-wrapper">
       <div class="off-canvas position-left" id="offCanvas" data-off-canvas>
         <%= render "navigation.html.erb",
-                   menus: Emitter::rails_report_specification.map { |x| x[:title] } %>
+                   menus: @reports.map { |x| x[:title] } %>
       </div>
       <div class="off-canvas-content" data-off-canvas-content>
         <button id="toggle-button" type="button" data-toggle="offCanvas">
@@ -52,24 +32,29 @@
         </button>
 
         <section class="main-section grid-container fluid">
-          <h1><%= options[:title] || "Log Sense Rails Log Report" %></h1>
+          <h1><%= options[:title] || "Log Sense #{@report_title} Log Report" %></h1>
 
-          <p><b>Input File(s):</b>  <%= data[:filenames].empty? ? "stdin" : data[:filenames].join(", ") %></p>
+          <p>
+            <b>Input File(s):</b>
+            <%= @data[:filenames].empty? ? "stdin" : @data[:filenames].join(", ") %>
+          </p>
 
           <div class="grid-x grid-padding-x">
             <article class="small-12 large-6 cell">
               <h2 id="<%= Emitter::slugify "Summary" %>">Summary</h2>
-              <%= render "summary.html.erb", data: data %>
+              <%= render "summary.html.erb", data: @data %>
             </article>
 
             <article class="small-12 large-6 cell">
               <h2 id="<%= Emitter::slugify "Summary" %>">Log Structure</h2>
-              <%= render "log_structure.html.erb", data: data %>
+              <%= render "log_structure.html.erb", data: @data %>
             </article>
 
             <% @reports.each_with_index do |report, index| %>
               <article class="cell <%= report[:col] || "small-12 large-6" %>" >
-                <h2 id="<%= Emitter::slugify report[:title] %>"><%= report[:title] %></h2>
+                <h2 id="<%= Emitter::slugify report[:title] %>">
+                  <%= report[:title] %>
+                </h2>
 
                 <%= render "report_data.html.erb", report: report, index: index %>
                 <% if report[:vega_spec] %>
@@ -95,21 +80,18 @@
 
             <article class="small-12 large-6 cell">
               <h2 id="<%= Emitter::slugify "Command Invocation" %>">Command Invocation</h2>
-              <%= render "command_invocation.html.erb", data: data, options: options %>
+              <%= render "command_invocation.html.erb", data: @data, options: options %>
             </article>
 
             <article class="small-12 large-6 cell">
               <h2 id="<%= Emitter::slugify "Performance" %>">Performance</h2>
-              <%=  render "performance.html.erb", data: data %>
+              <%=  render "performance.html.erb", data: @data %>
             </article>
           </div>
         </section>
       </div>
     </div>
 
-    <script type="text/javascript" src="js/vendor/what-input.js"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>    
-    <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/js/foundation.min.js" crossorigin="anonymous"></script>
     <script>
      $(document).foundation();
 

data/lib/log_sense/templates/{apache.txt.erb → report_txt.erb}

@@ -1,4 +1,4 @@
-* Apache Log Analysis
+* <%= @report_title %> Log Analysis
 
 <%= render "warning.txt.erb" %>
 

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "1.5.2"
+  VERSION = "1.6.0"
 end

data/lib/log_sense.rb

@@ -1,9 +1,19 @@
-require 'log_sense/version'
-require 'log_sense/options_parser'
-require 'log_sense/apache_log_line_parser'
-require 'log_sense/apache_log_parser'
-require 'log_sense/apache_data_cruncher'
-require 'log_sense/rails_log_parser'
-require 'log_sense/rails_data_cruncher'
-require 'log_sense/ip_locator'
-require 'log_sense/emitter'
+require "log_sense/version"
+
+require "log_sense/options_parser"
+require "log_sense/options_checker"
+
+require "log_sense/apache_log_parser"
+require "log_sense/rails_log_parser"
+
+require "log_sense/aggregator"
+require "log_sense/apache_aggregator"
+require "log_sense/rails_aggregator"
+
+require "log_sense/ip_locator"
+
+require "log_sense/report_shaper"
+require "log_sense/apache_report_shaper"
+require "log_sense/rails_report_shaper"
+
+require "log_sense/emitter"

data/log_sense.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = %q{Generate analytics for Apache and Rails log file.}
   spec.description   = %q{Generate analytics in HTML, txt, and SQLite format for Apache and Rails log files.}
-  spec.homepage      = "https://github.com/shair-tech/log_sense/log_sense"
+  spec.homepage      = "https://github.com/shair-tech/log_sense/"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.6.9")