log_sense 1.5.2 → 1.6.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.6.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-18 00:00:00.000000000 Z
+date: 2022-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser
@@ -124,20 +124,25 @@ files:
 - LICENSE.txt
 - README.org
 - Rakefile
-- apache-screenshot.png
 - bin/console
 - bin/setup
 - exe/log_sense
 - ip_locations/dbip-country-lite.sqlite3
 - lib/log_sense.rb
-- lib/log_sense/apache_data_cruncher.rb
+- lib/log_sense/aggregator.rb
+- lib/log_sense/apache_aggregator.rb
 - lib/log_sense/apache_log_line_parser.rb
 - lib/log_sense/apache_log_parser.rb
+- lib/log_sense/apache_report_shaper.rb
 - lib/log_sense/emitter.rb
 - lib/log_sense/ip_locator.rb
+- lib/log_sense/options_checker.rb
 - lib/log_sense/options_parser.rb
-- lib/log_sense/rails_data_cruncher.rb
+- lib/log_sense/rails_aggregator.rb
 - lib/log_sense/rails_log_parser.rb
+- lib/log_sense/rails_report_shaper.rb
+- lib/log_sense/report_shaper.rb
+- lib/log_sense/templates/_cdn_links.html.erb
 - lib/log_sense/templates/_command_invocation.html.erb
 - lib/log_sense/templates/_command_invocation.txt.erb
 - lib/log_sense/templates/_log_structure.html.erb
@@ -146,25 +151,26 @@ files:
 - lib/log_sense/templates/_output_table.txt.erb
 - lib/log_sense/templates/_performance.html.erb
 - lib/log_sense/templates/_performance.txt.erb
+- lib/log_sense/templates/_rails.css.erb
 - lib/log_sense/templates/_report_data.html.erb
 - lib/log_sense/templates/_stylesheet.css
 - lib/log_sense/templates/_summary.html.erb
 - lib/log_sense/templates/_summary.txt.erb
 - lib/log_sense/templates/_warning.txt.erb
-- lib/log_sense/templates/apache.html.erb
-- lib/log_sense/templates/apache.txt.erb
-- lib/log_sense/templates/rails.html.erb
-- lib/log_sense/templates/rails.txt.erb
+- lib/log_sense/templates/report_html.erb
+- lib/log_sense/templates/report_txt.erb
 - lib/log_sense/version.rb
 - log_sense.gemspec
 - sample_logs/safety-critical_org.log
 - sample_logs/spmbook_com.log
-homepage: https://github.com/shair-tech/log_sense/log_sense
+- screenshots/apache-screenshot.png
+- screenshots/rails-screenshot.png
+homepage: https://github.com/shair-tech/log_sense/
 licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org/
-  homepage_uri: https://github.com/shair-tech/log_sense/log_sense
+  homepage_uri: https://github.com/shair-tech/log_sense/
   source_code_uri: https://github.com/shair-tech/log_sense/
   changelog_uri: https://github.com/shair-tech/log_sense/blob/main/CHANGELOG.org
 post_install_message:

data/lib/log_sense/apache_data_cruncher.rb

@@ -1,147 +0,0 @@
-module LogSense
-  module ApacheDataCruncher
-    #
-    # take a sqlite3 database and analyze data
-    #
-    # @ variables are automatically put in the returned data
-    #
-
-    def self.crunch db, options = { limit: 900 }
-      first_day_s = db.execute "SELECT datetime from LogLine order by datetime limit 1"
-      last_day_s  = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
-
-      # make first and last day into dates or nil
-      @first_day = first_day_s&.first&.first ? Date.parse(first_day_s[0][0]) : nil
-      @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
-
-      @total_days = 0
-      @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
-
-      @source_files   = db.execute 'SELECT distinct(source_file) from LogLine'
-
-      @log_size       = db.execute 'SELECT count(datetime) from LogLine'
-      @log_size       = @log_size[0][0]
-
-      @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
-      @selfpolls_size = @selfpolls_size[0][0]
-
-      @crawlers_size  = db.execute 'SELECT count(datetime) from LogLine where bot == 1'
-      @crawlers_size = @crawlers_size[0][0]
-
-      @first_day_requested = options[:from_date]
-      @last_day_requested = options[:to_date]
-
-      @first_day_in_analysis = date_intersect options[:from_date], @first_day, :max
-      @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
-
-      @total_days_in_analysis = 0
-      if @first_day_in_analysis && @last_day_in_analysis
-        @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
-      end
-
-      #
-      # generate the where clause corresponding to the command line options to filter data
-      #
-      filter = [
-        (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
-        (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
-        (options[:only_crawlers] ? 'bot == 1' : nil),
-        (options[:ignore_crawlers] ? 'bot == 0' : nil),
-        (options[:no_selfpolls] ? "ip != '::1'" : nil),
-        'true'
-      ].compact.join " and "
-
-      mega = 1024 * 1024
-      giga = mega * 1024
-      tera = giga * 1024
- 
-      # in alternative to sum(size)
-      human_readable_size = <<-EOS
-      CASE
-      WHEN sum(size) <  1024 THEN sum(size) || ' B' 
-      WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
-      WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
-      WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
-      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB'
-      END AS size
-      EOS
-
-      human_readable_day = <<-EOS
-        case cast (strftime('%w', datetime) as integer)
-          when 0 then 'Sunday'
-          when 1 then 'Monday'
-          when 2 then 'Tuesday'
-          when 3 then 'Wednesday'
-          when 4 then 'Thursday'
-          when 5 then 'Friday'
-          else 'Saturday'
-        end as dow
-      EOS
-
-      @total_hits = db.execute "SELECT count(datetime) from LogLine where #{filter}"
-      @total_hits = @total_hits[0][0]
-
-      @total_unique_visits = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{filter}"
-      @total_unique_visits = @total_unique_visits[0][0]
-
-      @total_size = db.execute "SELECT #{human_readable_size} from LogLine where #{filter}"
-      @total_size = @total_size[0][0]
-
-      @daily_distribution = db.execute "SELECT date(datetime), #{human_readable_day}, count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
-      @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"
-
-      good_statuses = "(status like '2%' or status like '3%')"
-      bad_statuses = "(status like '4%' or status like '5%')"
-      html_page = "(extension like '.htm%')"
-      non_html_page = "(extension not like '.htm%')"
-
-      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{non_html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-
-      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-
-      @missed_pages_by_ip = db.execute "SELECT ip, path, status from LogLine where #{bad_statuses} and #{html_page} and #{filter} limit #{options[:limit]}"
-      @missed_resources_by_ip = db.execute "SELECT ip, path, status from LogLine where #{bad_statuses} and #{filter} limit #{options[:limit]}"
-
-      @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
-
-      @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"
-      @by_day_3xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '3' and #{filter} group by date(datetime)"
-      @by_day_2xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '2' and #{filter} group by date(datetime)"
-
-      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
-        [x[0], x[1].map { |y| y[1] }].flatten
-      }
-
-      @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
-      @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
-
-      @combined_platforms = db.execute "SELECT browser, platform, ip, count(datetime), #{human_readable_size} from LogLine where #{filter} group by browser, platform, ip order by count(datetime) desc limit #{options[:limit]}"
-
-      @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
-      
-      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
-
-      @streaks = db.execute 'SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime'
-      data = {}
-
-      instance_variables.each do |variable|
-        var_as_symbol = variable.to_s[1..].to_sym
-        data[var_as_symbol] = instance_variable_get(variable)
-      end
-
-      data
-    end
-
-    def self.date_intersect(date1, date2, method)
-      if date1 && date2
-        [date1, date2].send(method)
-      elsif date1
-        date1
-      else
-        date2
-      end
-    end
-  end
-end

data/lib/log_sense/rails_data_cruncher.rb

@@ -1,141 +0,0 @@
-module LogSense
-  module RailsDataCruncher
-
-    #
-    # take a sqlite3 database and analyze data
-    #
-    # @ variables are automatically put in the returned data
-    #
-
-    def self.crunch db, options = { limit: 900 }
-      first_day_s = db.execute "SELECT started_at from Event where started_at not NULL order by started_at limit 1"
-      # we could use ended_at to cover the full activity period, but I prefer started_at
-      # with the meaning that the monitor event initiation
-      last_day_s  = db.execute "SELECT started_at from Event order by started_at desc limit 1"
-
-      # make first and last day into dates or nil
-      # TODO: bug possible value here: [[nil]], which is not empty
-      @first_day = first_day_s&.first&.first ? Date.parse(first_day_s[0][0]) : nil
-      @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
-
-      @total_days = 0
-      @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
-
-      # TODO should also look into Error
-      @source_files = db.execute "SELECT distinct(source_file) from Event"
-
-      @log_size = db.execute "SELECT count(started_at) from Event"
-      @log_size = @log_size[0][0]
-
-      # TODO: I should make the names of events/size/etc uniform betweeen Apache and Rails Logs
-      # SAME AS ABOVE
-      @total_hits = @log_size
-
-      # SAME AS ABOVE (but log_size is wrong in the case of Rails
-      # logs, since an event takes more than one line)
-      @events = db.execute "SELECT count(started_at) from Event"
-      @events = @events[0][0]
-
-      @first_day_requested = options[:from_date]
-      @last_day_requested = options[:to_date]
-
-      @first_day_in_analysis = date_intersect options[:from_date], @first_day, :max
-      @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
-
-      @total_days_in_analysis = 0
-      if @first_day_in_analysis and @last_day_in_analysis
-        @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
-      end
-
-      #
-      # generate the where clause corresponding to the command line options to filter data
-      #
-      filter = [
-        (options[:from_date] ? "date(started_at) >= '#{options[:from_date]}'" : nil),
-        (options[:to_date] ? "date(started_at) <= '#{options[:to_date]}'" : nil),
-        "true"
-      ].compact.join " and "
-
-      mega = 1024 * 1024
-      giga = mega * 1024
-      tera = giga * 1024
-      
-      # in alternative to sum(size)
-      human_readable_size = <<-EOS
-      CASE 
-      WHEN sum(size) <  1024 THEN sum(size) || ' B' 
-      WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
-      WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
-      WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
-      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
-      END AS size
-      EOS
-
-      human_readable_day = <<-EOS
-        case cast (strftime('%w', started_at) as integer)
-          when 0 then 'Sunday'
-          when 1 then 'Monday'
-          when 2 then 'Tuesday'
-          when 3 then 'Wednesday'
-          when 4 then 'Thursday'
-          when 5 then 'Friday'
-          else 'Saturday'
-        end as dow
-      EOS
-
-      @total_events = db.execute "SELECT count(started_at) from Event where #{filter}"
-
-      @total_unique_visits = db.execute "SELECT count(distinct(unique_visitor)) from Event where #{filter}"
-      @total_unique_visits = @total_unique_visits[0][0]
-
-      @daily_distribution = db.execute "SELECT date(started_at), #{human_readable_day}, count(started_at) from Event where #{filter} group by date(started_at)"
-      @time_distribution = db.execute "SELECT strftime('%H', started_at), count(started_at) from Event where #{filter} group by strftime('%H', started_at)"
-
-      @statuses = db.execute "SELECT status, count(status) from Event where #{filter} group by status order by status"
-
-      @by_day_5xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '5' and #{filter} group by date(started_at)"
-      @by_day_4xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '4' and #{filter} group by date(started_at)"
-      @by_day_3xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '3' and #{filter} group by date(started_at)"
-      @by_day_2xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '2' and #{filter} group by date(started_at)"
-
-      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx + @by_day_5xx).group_by { |x| x[0] }.to_a.map { |x|
-        [x[0], x[1].map { |y| y[1] }].flatten
-      }
-
-      @ips = db.execute "SELECT ip, count(ip) from Event where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
-
-      @streaks = db.execute 'SELECT ip, substr(started_at, 1, 10), url from Event order by ip, started_at'
-      data = {}
-
-      @performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
-
-      @fatal = db.execute ("SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE exit_status == 'F'") || [[]]
-
-      @internal_server_error = (db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), status, ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE status is 500") || [[]]
-
-      @error = (db.execute "SELECT log_id, context, description, count(log_id) from Error GROUP BY description") || [[]]
-      
-      data = {}
-      self.instance_variables.each do |variable|
-        var_as_symbol = variable.to_s[1..-1].to_sym
-        data[var_as_symbol] = eval(variable.to_s)
-      end
-      data
-    end
-
-    private
-
-    def self.date_intersect date1, date2, method
-      if date1 and date2
-        [date1, date2].send(method)
-      elsif date1
-        date1
-      else
-        date2
-      end
-    end
-
-
-  end
-end
-

data/lib/log_sense/templates/apache.html.erb

@@ -1,115 +0,0 @@
-<!doctype html>
-<html class="no-js" lang="en">
-  <head>
-    <title>
-      <%= options[:title] || "Log Sense: #{data[:filenames].empty? ? "stdin" : data[:filenames].join(", ")}" %>
-    </title>
-
-    <meta charset="utf-8" />
-    <meta http-equiv="x-ua-compatible" content="ie=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-
-    <meta name="author" content="Shair.Tech">
-    <meta name="description" content="Analysis of <%= data[:filenames].join(', ') %>">
-
-    <link rel="preconnect" href="https://fonts.googleapis.com">
-    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-    <link href="https://fonts.googleapis.com/css2?family=Fira+Sans:wght@300;700&display=swap" rel="stylesheet">
-    
-    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/foundicons/3.0.0/foundation-icons.min.css">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/css/foundation.min.css">
-    <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.css"/>
-
-    <script type="text/javascript" src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
-    <script type="text/javascript" src="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.js"></script>
-
-    <script src="https://cdn.jsdelivr.net/npm/vega@5.21.0"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega-lite@5.2.0"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega-embed@6.20.2"></script>
-
-    <style>
-     <%= render "stylesheet.css" %>
-    </style>
-  </head>
-
-  <body>
-    <div class="off-canvas-wrapper">
-      <div class="off-canvas position-left" id="offCanvas" data-off-canvas>
-        <%= render "navigation.html.erb",
-                   menus: Emitter::apache_report_specification.map { |x| x[:title] } %>
-      </div>
-      <div class="off-canvas-content" data-off-canvas-content>
-        <button id="toggle-button" type="button" data-toggle="offCanvas">
-          &#9776;
-        </button>
-
-        <section class="main-section grid-container fluid">
-          <h1><%= options[:title] || "Log Sense Apache Log Report" %></h1>
-
-          <p><b>Input File(s):</b>  <%= data[:filenames].empty? ? "stdin" : data[:filenames].join(", ") %></p>
-
-          <div class="grid-x grid-padding-x">
-            <article class="small-12 large-6 cell">
-              <h2 id="<%= Emitter::slugify "Summary" %>">Summary</h2>
-              <%= render "summary.html.erb", data: data %>
-            </article>
-
-            <article class="small-12 large-6 cell">
-              <h2 id="<%= Emitter::slugify "Summary" %>">Log Structure</h2>
-              <%= render "log_structure.html.erb", data: data %>
-            </article>
-
-            <% @reports.each_with_index do |report, index| %>
-              <article class="<%= report[:col] || "small-12 large-6" %> cell">
-                <h2 id="<%= Emitter::slugify report[:title] %>"><%= report[:title] %></h2>
-
-                <%= render "report_data.html.erb", report: report, index: index %>
-                <% if report[:vega_spec] %>
-                  <div id="<%= "plot-#{index}" %>" class="plot-canvas">
-                  </div>
-                  <script>
-                   plot_spec_<%= index %> = Object.assign(
-                     <%= report[:vega_spec].to_json %>,
-                     { "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
-                       width: "container",
-                       description: "<%= report[:title] %>",
-                       data: {
-                         values: data_<%= index %>
-                       },
-                   });
-                   vegaEmbed('#<%= "plot-#{index}"%>', plot_spec_<%= index %>);
-                  </script>
-                <% end %>
-
-                <%= render "output_table.html.erb", report: report, index: index %>
-              </article>
-            <% end %>
-
-            <article class="small-12 large-6 cell">
-              <h2 id="<%= Emitter::slugify "Command Invocation" %>">Command Invocation</h2>
-              <%= render "command_invocation.html.erb", data: data, options: options %>
-            </article>
-
-            <article class="small-12 large-6 cell">
-              <h2 id="<%= Emitter::slugify "Performance" %>">Performance</h2>
-              <%=  render "performance.html.erb", data: data %>
-            </article>
-          </div>
-        </section>
-      </div>
-    </div>
-
-    <script type="text/javascript" src="js/vendor/what-input.js"></script>
-    <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>    
-    <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/js/foundation.min.js" crossorigin="anonymous"></script>
-    <script>
-     $(document).foundation();
-
-     $(document).ready(function () {
-       $('.data-table').each(function () {
-         $(this).DataTable();
-       });
-     });
-    </script>
-  </body>
-</html>

data/lib/log_sense/templates/rails.txt.erb

@@ -1,22 +0,0 @@
-* Rails Log Analysis
-
-<%= render "warning.txt.erb" %>
-
-** Summary
-
-<%= render "summary.txt.erb", data: data %>
-
-<% @reports.reject { |x| x[:report] == :html }.each do |report| %>
-** <%= report[:title] %>
-
-<%= render "output_table.txt.erb", report: report, data: data %>
-<% end %>
-
-** Command Invocation
-
-<%= render 'command_invocation.txt.erb', data: data %>
-
-** Performance
-
-<%= render 'performance.txt.erb', data: data %>
-