log_sense 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ef1d35932ba8c7fe6e636f6cb507ea99fd6b6026170d62207dfa2606d62bbcb
-  data.tar.gz: 725dc6e51ace6c9cbd366e3888387ef8f246327c9e1036c66d2eb6351b1f0791
+  metadata.gz: '0128717b2ba709bc5dfbb7b755762a757c575ad4307159910cc894aaa3b88f42'
+  data.tar.gz: e05054b8eee79a439f5b077e60bc0d95a3e7706c550853333d7d631c458abf91
 SHA512:
-  metadata.gz: 592e80cd56f4740cc4003b494c9da960de228025377d630d070a08a495cffac7fd41850d469189510def48f9940700251bd84a141def9b1b15b32fbed967261f
-  data.tar.gz: 1d802e95fd58c66c4904843478c5ccc8ffc1ce43e87dd199e5b21efc89bcff436b227c646d2e5a386bee22a3d1c67c0341c25fcb4058e48526560d05984f0148
+  metadata.gz: 9d9e3dc495f7479292ae96d1bf6298f531258cae74df47ff705aea9613880b0d50aa6a19328f70685484ad1c606bd12a8fb7c87632fe5c9cbefefe4893d9bb4d
+  data.tar.gz: b417049bcc119ed82ab4c33d007e15804ba85b2485308ca28448fba512814fc5e8b8b215310bfb5c8108fcdc87292322eefc6826b18718fcbc7fced29eea77cb

data/CHANGELOG.org

@@ -2,6 +2,21 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.5.0
+
+- [User] Present Unique Visits / day as integer
+- [User] Added Country and Streaks report for rails
+- [User] Changed Streak report in Apache
+
+- [Gem] Updated DBIP
+- [Gem] Updated Bundle  
+
+- [Code] Refactored all reports, so that they are specified
+  in the same way  
+- [Code] Refactor warning message in textual reports
+- [Code] Build HTML menu for report specification
+- [Code] Various refactoring passes on the code
+
 * 1.4.1
 
 - [User] New textual report for Apache

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.3.1)
+    log_sense (1.4.2)
       browser
       ipaddr
       iso_country_codes
@@ -13,9 +13,9 @@ GEM
   specs:
     browser (5.3.1)
     byebug (11.1.3)
-    ipaddr (1.2.3)
+    ipaddr (1.2.4)
     iso_country_codes (0.7.8)
-    minitest (5.14.4)
+    minitest (5.15.0)
     rake (12.3.3)
     sqlite3 (1.4.2)
     terminal-table (3.0.2)
@@ -32,4 +32,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.2.32
+   2.3.3

data/README.org

@@ -19,8 +19,6 @@ LogSense reports the following data:
 - OS, browsers, and devices
 - IP Country location, thanks to the DPIP lite country DB
 - Streaks: resources accessed by a given IP over time
-- Potential attacks: access to resources which are not meant to be
-  served by a web server serving static websites
 - Performance of Rails requests
  
 Filters from the command line allow to analyze specific periods and
@@ -33,6 +31,18 @@ And, of course, the compulsory screenshot:
 #+ATTR_HTML: :width 80%
 [[file:./apache-screenshot.png]]
 
+
+* An important word of warning
+
+[[https://owasp.org/www-community/attacks/Log_Injection][Log poisoning]] is a technique whereby attackers send requests with invalidated
+user input to forge log entries or inject malicious content into the logs.
+
+log_sense sanitizes entries of HTML reports, to try and protect from log
+poisoning.  *Log entries and URLs in SQLite3, however, are not sanitized*:
+they are stored and read from the log.  This is not, in general, an issue,
+unless you use the data from SQLite in environments in which URLs can be
+opened or code executed.
+
 * Motivation
 
 LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] (which
@@ -54,6 +64,7 @@ generated files are then made available on a private area on the web.
   gem install log_sense
   #+end_src
 
+
 * Usage
 
   #+begin_src bash :results raw output :wrap example
@@ -73,10 +84,11 @@ generated files are then made available on a private area on the web.
       -w, --width=WIDTH                Maximum width of URL and description columns in text reports
       -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
       -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
+          --verbose                    Inform about progress (prints to STDERR)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.4.1
+  This is version 1.5.0
 
   Output formats
   rails parsing can produce the following outputs:
@@ -96,6 +108,7 @@ log_sense -f apache -i access.log -t txt > access-data.txt
 log_sense -f rails -i production.log -t html -o performance.txt
 #+end_example
 
+
 * Change Log
 
 See the [[file:CHANGELOG.org][CHANGELOG]] file.
@@ -110,8 +123,8 @@ Concerning the outputs:
 - HTML reports use [[https://get.foundation/][Zurb Foundation]], [[https://www.datatables.net/][Data Tables]], and [[https://vega.github.io/vega-lite/][Vega Light]], which
   are all downloaded from a CDN
 - The textual format is compatible with [[https://orgmode.org/][Org Mode]] and can be further
-  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to (including HTML
-  and PDF)
+  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to, including HTML
+  and PDF, with the word of warning in the section above. 
 
 * Author and Contributors
 
@@ -119,8 +132,8 @@ Concerning the outputs:
 
 * Known Bugs
 
-No known bugs; an unknown number of unknown bugs.
-(See the open issues for the known bugs.)
+No known bugs; an unknown number of unknown bugs.  (See the open issues for
+the known bugs.)
 
 * License
 

data/Rakefile

@@ -9,7 +9,21 @@ end
 require_relative './lib/log_sense/ip_locator.rb'
 
 desc "Convert Geolocation DB to sqlite"
-task :dbip_to_sqlite3, [:filename] do |tasks, args|
-  filename = args[:filename]
-  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+task :dbip_to_sqlite3, [:year_month] do |tasks, args|
+  filename = "./ip_locations/dbip-country-lite-#{args[:year_month]}.csv"
+
+  if !File.exist? filename
+    puts "Error. Could not find: #{filename}"
+    puts
+    puts 'I see the following files:'
+    puts Dir.glob("ip_locations/dbip-country-lite*").map { |x| "- #{x}\n" }
+    puts ''
+    puts '1. Download (if necessary) a more recent version from: https://db-ip.com/db/download/ip-to-country-lite'
+    puts '2. Save downloaded file to ip_locations/'
+    puts '3. Relaunch with YYYY-MM'
+
+    exit
+  else
+    LogSense::IpLocator::dbip_to_sqlite filename
+  end
 end

data/exe/log_sense

@@ -12,7 +12,7 @@ require 'log_sense.rb'
 @output_file = @options[:output_file]
 
 if ARGV.map { |x| File.exist?(x) }.include?(false)
-  warn.puts "Error: input file(s) '#{ARGV.reject { |x| File.exist(x) }.join(', ')}' do not exist"
+  $stderr.puts "Error: input file(s) '#{ARGV.reject { |x| File.exist(x) }.join(', ')}' do not exist"
   exit 1
 end
 @input_files = ARGV.empty? ? [$stdin] : ARGV.map { |x| File.open(x, 'r') }
@@ -31,35 +31,45 @@ when 'rails'
   parser_klass = LogSense::RailsLogParser
   cruncher_klass = LogSense::RailsDataCruncher
 else
-  warn.puts "Error: input format #{@options[:input_format]} not understood."
+  $stderr.puts "Error: input format #{@options[:input_format]} not understood."
   exit 1
 end
 
+$stderr.puts "Parsing input files..." if @options[:verbose]
 @db = parser_klass.parse @input_files
 
 if @options[:output_format] == 'sqlite'
+  $stderr.puts "Saving to SQLite3..." if @options[:verbose]
   ddb = SQLite3::Database.new(@output_file || 'db.sqlite3')
   b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
   b.step(-1) #=> DONE
   b.finish
 else
+  $stderr.puts "Aggregating data..." if @options[:verbose]
   @data = cruncher_klass.crunch @db, @options
+
+  $stderr.puts "Geolocating..." if @options[:verbose]
   @data = LogSense::IpLocator.geolocate @data
 
+  $stderr.puts "Grouping by country..." if @options[:verbose]
+  country_col = @data[:ips][0].size - 1
+  @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+
   @ended_at = Time.now
   @duration = @ended_at - @started_at
 
   @data = @data.merge({
                         command: @command_line,
+                        filenames: ARGV,
                         log_files: @input_files,
                         started_at: @started_at,
                         ended_at: @ended_at,
                         duration: @duration,
                         width: @options[:width]
                       })
-
   #
   # Emit Output
   #
+  $stderr.puts "Emitting..." if @options[:verbose]
   puts LogSense::Emitter.emit @data, @options
 end

data/lib/log_sense/apache_data_cruncher.rb

@@ -17,15 +17,15 @@ module LogSense
       @total_days = 0
       @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
 
-      @source_files   = db.execute "SELECT distinct(source_file) from LogLine"
+      @source_files   = db.execute 'SELECT distinct(source_file) from LogLine'
 
-      @log_size       = db.execute "SELECT count(datetime) from LogLine"
+      @log_size       = db.execute 'SELECT count(datetime) from LogLine'
       @log_size       = @log_size[0][0]
 
       @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
       @selfpolls_size = @selfpolls_size[0][0]
 
-      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @crawlers_size  = db.execute 'SELECT count(datetime) from LogLine where bot == 1'
       @crawlers_size = @crawlers_size[0][0]
 
       @first_day_requested = options[:from_date]
@@ -35,7 +35,7 @@ module LogSense
       @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
 
       @total_days_in_analysis = 0
-      if @first_day_in_analysis and @last_day_in_analysis
+      if @first_day_in_analysis && @last_day_in_analysis
         @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
       end
 
@@ -45,24 +45,24 @@ module LogSense
       filter = [
         (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
         (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
-        (options[:only_crawlers] ? "bot == 1" : nil),
-        (options[:ignore_crawlers] ? "bot == 0" : nil),
+        (options[:only_crawlers] ? 'bot == 1' : nil),
+        (options[:ignore_crawlers] ? 'bot == 0' : nil),
         (options[:no_selfpolls] ? "ip != '::1'" : nil),
-        "true"
+        'true'
       ].compact.join " and "
 
       mega = 1024 * 1024
       giga = mega * 1024
       tera = giga * 1024
-      
+ 
       # in alternative to sum(size)
       human_readable_size = <<-EOS
-      CASE 
+      CASE
       WHEN sum(size) <  1024 THEN sum(size) || ' B' 
       WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
       WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
       WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
-      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
+      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB'
       END AS size
       EOS
 
@@ -117,20 +117,19 @@ module LogSense
       
       @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @streaks = db.execute 'SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime'
       data = {}
 
-      self.instance_variables.each do |variable|
-        var_as_symbol = variable.to_s[1..-1].to_sym
-        data[var_as_symbol] = eval(variable.to_s)
+      instance_variables.each do |variable|
+        var_as_symbol = variable.to_s[1..].to_sym
+        data[var_as_symbol] = instance_variable_get(variable)
       end
+
       data
     end
 
-    private
-
-    def self.date_intersect date1, date2, method
-      if date1 and date2
+    def self.date_intersect(date1, date2, method)
+      if date1 && date2
         [date1, date2].send(method)
       elsif date1
         date1
@@ -140,4 +139,3 @@ module LogSense
     end
   end
 end
-

data/lib/log_sense/apache_log_line_parser.rb

@@ -44,7 +44,7 @@ module LogSense
 
     attr_reader :format
 
-    def initialize 
+    def initialize
       @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
     end
 

data/lib/log_sense/apache_log_parser.rb

@@ -82,7 +82,7 @@ module LogSense
               line_number
             )
           rescue StandardError => e
-            warn.puts e.message
+            $stderr.puts e.message
           end
         end
       end

data/lib/log_sense/emitter.rb

@@ -1,8 +1,8 @@
+# coding: utf-8
 require 'terminal-table'
 require 'json'
 require 'erb'
 require 'ostruct'
-
 module LogSense
   #
   # Emit Data
@@ -31,9 +31,7 @@ module LogSense
       end
     end
 
-    private_class_method
-
-    def self.render(template, vars)
+    def self.render(template, vars = {})
       @template = File.join(File.dirname(__FILE__), 'templates', "_#{template}")
       erb_template = File.read @template
       ERB.new(erb_template).result(OpenStruct.new(vars).instance_eval { binding })
@@ -42,11 +40,11 @@ module LogSense
     def self.escape_javascript(string)
       js_escape_map = {
         '<' => '&lt;',
-        '</' => '&lt;\/',
-        '\\' => '\\\\',
+        '</' => '&lt;/',
         '\r\n' => '\\r\\n',
         '\n' => '\\n',
         '\r' => '\\r',
+        '\\' => ' \\\\',
         '"' => ' \\"',
         "'" => " \\'",
         '`' => ' \\`',
@@ -73,7 +71,9 @@ module LogSense
     # - width width to set
     def self.shorten(data, heading, width)
       # indexes of columns which have to be shortened
-      to_shorten = %w[URL Description Path].map { |x| heading.index x }.compact
+      keywords = %w[URL Referers Description Path]
+      to_shorten = keywords.map { |x| heading.index x }.compact
+
       if width.nil? || to_shorten.empty? || data[0].nil?
         data
       else
@@ -96,7 +96,7 @@ module LogSense
     #   column_alignment: specification of column alignments (works for txt reports)
     #   vega_spec: specifications for Vega output
     #   datatable_options: specific options for datatable
-    def self.apache_report_specification(data)
+    def self.apache_report_specification(data = {})
       [
         { title: 'Daily Distribution',
           header: %w[Day DOW Hits Visits Size],
@@ -326,6 +326,19 @@ module LogSense
           column_alignment: %i[left right right right left],
           rows: data[:ips]
         },
+        {
+          title: 'Countries',
+          header: %w[Country Hits Visits IPs],
+          column_alignment: %i[left right right left],
+          rows: data[:countries]&.map do |k, v|
+            [
+              k,
+              v.map { |x| x[1] }.inject(&:+),
+              v.map { |x| x[2] }.inject(&:+),
+              v.map { |x| x[0] }.join(' ')
+            ]
+          end
+        },
         {
           title: 'Referers',
           header: %w[Referers Hits Visits Size],
@@ -333,10 +346,27 @@ module LogSense
           rows: data[:referers],
           col: 'small-12 cell'
         },
+        {
+          title: 'Streaks',
+          report: :html,
+          header: ['IP', 'Date', 'Total HTML', 'Total Other', 'HTML', 'Other'],
+          column_alignment: %i[left left right right left left],
+          rows: data[:streaks]&.group_by { |x| [x[0], x[1]] }&.map do |k, v|
+            [
+              k[0],
+              k[1],
+              v.map { |x| x[2] }.compact.select { |x| x.match(/\.html?$/) }.size,
+              v.map { |x| x[2] }.compact.reject { |x| x.match(/\.html?$/) }.size,
+              v.map { |x| x[2] }.compact.select { |x| x.match(/\.html?$/) }.join(' ■ '),
+              v.map { |x| x[2] }.compact.reject { |x| x.match(/\.html?$/) }.join(' ■ ')
+            ]
+          end,
+          col: 'small-12 cell'
+        }
       ]
     end
 
-    def self.rails_report_specification(data)
+    def self.rails_report_specification(data = {})
       [
         {
           title: "Daily Distribution",
@@ -469,27 +499,54 @@ module LogSense
           header: %w[Date IP URL Description Log ID],
           column_alignment: %i[left left left left left],
           rows: data[:fatal],
-          col: "small-12 cell"
+          col: 'small-12 cell'
         },
         {
-          title: "Internal Server Errors",
+          title: 'Internal Server Errors',
           header: %w[Date Status IP URL Description Log ID],
           column_alignment: %i[left left left left left left],
           rows: data[:internal_server_error],
-          col: "small-12 cell"
+          col: 'small-12 cell'
         },
         {
-          title: "Errors",
+          title: 'Errors',
           header: %w[Log ID Context Description Count],
           column_alignment: %i[left left left left],
           rows: data[:error],
-          col: "small-12 cell"
+          col: 'small-12 cell'
         },
         {
-          title: "IPs",
+          title: 'IPs',
           header: %w[IPs Hits Country],
           column_alignment: %i[left right left],
           rows: data[:ips]
+        },
+        {
+          title: 'Countries',
+          header: %w[Country Hits IPs],
+          column_alignment: %i[left right left],
+          rows: data[:countries]&.map do |k, v|
+            [
+              k,
+              v.map { |x| x[1] }.inject(&:+),
+              v.map { |x| x[0] }.join(' ■ ')
+            ]
+          end
+        },
+        {
+          title: 'Streaks',
+          report: :html,
+          header: %w[IP Date Total Resources],
+          column_alignment: %i[left left right right left left],
+          rows: data[:streaks]&.group_by { |x| [x[0], x[1]] }&.map do |k, v|
+            [
+              k[0],
+              k[1],
+              v.size,
+              v.map { |x| x[2] }.join(' ■ ')
+            ]
+          end,
+          col: 'small-12 cell'
         }
       ]
     end

data/lib/log_sense/ip_locator.rb

@@ -4,19 +4,22 @@ require 'ipaddr'
 require 'iso_country_codes'
 
 module LogSense
+  #
+  # Populate table of IP Locations from dbip-country-lite
+  #
   module IpLocator
-    DB_FILE = File.join(File.dirname(__FILE__), "..", "..", "ip_locations", "dbip-country-lite.sqlite3")
+    DB_FILE = File.join(File.dirname(__FILE__), '..', '..', 'ip_locations', 'dbip-country-lite.sqlite3')
 
-    def self.dbip_to_sqlite db_location
-      db = SQLite3::Database.new ":memory:"
-      db.execute "CREATE TABLE ip_location (
+    def self.dbip_to_sqlite(db_location)
+      db = SQLite3::Database.new ':memory:'
+      db.execute 'CREATE TABLE ip_location (
         from_ip_n INTEGER,
         from_ip TEXT,
         to_ip TEXT,
         country_code TEXT
-      )"
+      )'
 
-      ins = db.prepare "INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)"
+      ins = db.prepare 'INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)'
       CSV.foreach(db_location) do |row|
         ip = IPAddr.new row[0]
         ins.execute(ip.to_i, row[0], row[1], row[2])
@@ -33,29 +36,33 @@ module LogSense
       SQLite3::Database.new DB_FILE
     end
 
-    def self.locate_ip ip, db
-      return if not ip
+    def self.locate_ip(ip, db)
+      return unless ip
 
-      ip_n = IPAddr.new(ip).to_i
-      res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
+      query = db.prepare 'SELECT * FROM ip_location where from_ip_n <= ? order by from_ip_n desc limit 1'
       begin
-        IsoCountryCodes.find(res[0][3]).name
-      rescue
-        res[0][3]
+        ip_n = IPAddr.new(ip).to_i
+        result_set = query.execute ip_n
+        country_code = result_set.map { |x| x[3] }[0]
+        IsoCountryCodes.find(country_code).name
+      rescue IPAddr::InvalidAddressError
+        'INVALID IP'
+      rescue IsoCountryCodes::UnknownCodeError
+        country_code
       end
     end
 
     #
     # add country code to data[:ips]
     #
-    def self.geolocate data
-      @location_db = IpLocator::load_db
-      data[:ips].each do |ip|
-        country_code = IpLocator::locate_ip ip[0], @location_db
-        ip << country_code
+    def self.geolocate(data)
+      @location_db = IpLocator.load_db
+
+      data[:ips].each do |line|
+        country_code = IpLocator.locate_ip line[0], @location_db
+        line << country_code
       end
       data
     end
-
   end
 end

data/lib/log_sense/options_parser.rb

@@ -59,6 +59,10 @@ module LogSense
           args[:no_selfpoll] = true
         end
 
+        opts.on('--verbose', 'Inform about progress (prints to STDERR)') do
+          args[:verbose] = true
+        end
+
         opts.on('-v', '--version', 'Prints version information') do
           puts "log_sense version #{LogSense::VERSION}"
           puts 'Copyright (C) 2021 Shair.Tech'
@@ -96,6 +100,7 @@ module LogSense
       args[:ignore_crawlers] ||= false
       args[:only_crawlers] ||= false
       args[:no_selfpoll] ||= false
+      args[:verbose] ||= false
 
       args
     end

data/lib/log_sense/rails_data_cruncher.rb

@@ -104,6 +104,9 @@ module LogSense
 
       @ips = db.execute "SELECT ip, count(ip) from Event where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
+      @streaks = db.execute 'SELECT ip, substr(started_at, 1, 10), url from Event order by ip, started_at'
+      data = {}
+
       @performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
 
       @fatal = db.execute ("SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE exit_status == 'F'") || [[]]

data/lib/log_sense/templates/_navigation.html.erb

@@ -0,0 +1,21 @@
+<nav>
+  <h2>Navigation</h2>
+  <ul class="no-bullet">
+    <% (["Summary", "Log Structure"] +
+        menus +
+        ["Command Invocation", "Performance"]).each do |item| %>
+      <li class="nav-item">
+        <a href="#<%= Emitter::slugify item %>" data-close>
+          <%= item %>
+        </a>
+      </li>
+    <% end %>
+  </ul>
+
+  <p>
+    Generated by
+    <a href="https://github.com/avillafiorita/log_sense">LogSense</a> <br />
+    on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
+    <a href='https://db-ip.com'>IP Geolocation by DB-IP</a>
+  </p>
+</nav>

data/lib/log_sense/templates/_output_table.txt.erb

@@ -6,8 +6,9 @@ shortened = Emitter::shorten(report[:rows], report[:header], data[:width])
 table = Terminal::Table.new headings: report[:header], rows: shortened
 table.style = { border_i: "|" }
 columns = report[:header].size - 1
-(0..columns).map { |i| table.align_column(i, report[:column_alignment][i] || :left) }
-
+(0..columns).map do |i|
+  table.align_column(i, report[:column_alignment][i] || :left)
+end
 # return it
 table
 %>

data/lib/log_sense/templates/_summary.html.erb

@@ -18,7 +18,7 @@
   </li>
   <li class="stats-list-negative">
     <% days = data[:last_day_in_analysis] - data[:first_day_in_analysis] %>
-    <%= days > 0 ? "%.2f" % (data[:total_unique_visits] / days.to_f) : "N/A" %>
+    <%= days > 0 ? "%d" % (data[:total_unique_visits] / days) : "N/A" %>
     <span class="stats-list-label">Unique Visits / Day</span>
   </li>
   <li class="stats-list-negative">

data/lib/log_sense/templates/_summary.txt.erb

@@ -5,7 +5,7 @@ table = Terminal::Table.new rows: [
   ["Days", data[:total_days_in_analysis]],
   ["Hits", data[:total_hits]],
   ["Unique Visits", data[:total_unique_visits]],
-  ["Unique Visits / Day", data[:total_days_in_analysis] > 0 ? "%.2f" % (data[:total_unique_visits] / data[:total_days_in_analysis].to_f) : "N/A"],
+  ["Unique Visits / Day", data[:total_days_in_analysis] > 0 ? "%d" % (data[:total_unique_visits] / data[:total_days_in_analysis]) : "N/A"],
   ["Hits/Unique Visitor", data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A"]
 ]
 table.style = { border_i: "|" }

data/lib/log_sense/templates/_warning.txt.erb

@@ -0,0 +1 @@
+>>>> URLs IN THIS FILE ARE NOT SANITIZED AND MIGHT BE UNSAFE TO OPEN <<<<

data/lib/log_sense/templates/apache.html.erb

@@ -1,7 +1,9 @@
 <!doctype html>
 <html class="no-js" lang="en">
   <head>
-    <title><%= options[:title] || "Log Sense: #{data[:log_file]}" %></title>
+    <title>
+      <%= options[:title] || "Log Sense: #{data[:filenames].empty? ? "stdin" : data[:filenames].join(", ")}" %>
+    </title>
 
     <meta charset="utf-8" />
     <meta http-equiv="x-ua-compatible" content="ie=edge">
@@ -150,42 +152,11 @@
   <body>
     <div class="off-canvas-wrapper">
       <div class="off-canvas position-left" id="offCanvas" data-off-canvas>
-        <nav>
-          <h2>Navigation</h2>
-          <ul class="no-bullet">
-                <% [ "Summary",
-                  "Log Structure",
-                  "Daily Distribution",
-                  "Time Distribution",
-                  "20_ and 30_ on HTML pages",
-                  "20_ and 30_ on other resources",
-                  "40_ and 50_ on HTML pages",
-                  "40_ and 50_ on other Resources",
-                  "Statuses",
-                  "Daily Statuses",
-                  "Browsers",
-                  "Platforms",
-                  "Referers",
-                  "IPs",
-                  "Geolocation",
-                  "Streaks",
-                  "Command Invocation",
-                  "Performance"
-            ].each do |item| %>
-                  <li class="nav-item">
-                    <a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
-              </li>
-            <% end %>
-          </ul>
-
-          <p>
-            Generated by
-            <a href="https://github.com/avillafiorita/log_sense">LogSense</a> <br />
-            on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
-            <a href='https://db-ip.com'>IP Geolocation by DB-IP</a>
-          </p>
-        </nav>
+        <%= render "navigation.html.erb",
+                   menus: Emitter::apache_report_specification.map { |x| x[:title] }  +
+                   ["Streaks", "Command Invocation", "Performance"] %>
       </div>
+      
       <div class="off-canvas-content grid-container grid-x fluid" data-off-canvas-content>
         <div data-sticky-container>
           <div class="sticky" data-sticky data-margin-top="0">
@@ -196,14 +167,14 @@
         </div>
 
         <section class="main-section">
-          <h1><%= options[:title] || "Log Sense: #{data[:log_file]}" %></h1>
+          <h1><%= options[:title] || "Log Sense Apache Log Report" %></h1>
 
-          <p><b>Input File:</b>  <%= (data[:log_file] || "stdin") %></p>
+          <p><b>Input File(s):</b>  <%= data[:filenames].empty? ? "stdin" : data[:filenames].join(", ") %></p>
 
           <div class="grid-x grid-margin-x">
             <article class="card small-12 large-6 cell">
               <div class="card-divider">
-                <h2 id="summary">Summary</h2>
+                <h2 id="<%= Emitter::slugify "Summary" %>">Summary</h2>
               </div>
               <div class="card-section">
                 <%= render "summary.html.erb", data: data %>
@@ -212,7 +183,7 @@
 
             <article class="card cell small-12 large-6">
               <div class="card-divider">
-                <h2 id="log-structure">Log Structure</h2>
+                <h2 id="<%= Emitter::slugify "Summary" %>">Log Structure</h2>
               </div>
               <div class="card-section">
                 <%= render "log_structure.html.erb", data: data %>
@@ -224,7 +195,7 @@
             <% @reports.each_with_index do |report, index| %>
               <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
                 <div class="card-divider">
-                  <h2 id="<%= report[:title].downcase.gsub(' ', '-') %>">
+                  <h2 id="<%= Emitter::slugify report[:title] %>">
                     <%= report[:title] %>
                   </h2>
                 </div>
@@ -252,101 +223,10 @@
             <% end %>
           </div>
 
-          <article class="card">
-            <div class="card-divider">
-              <h2 id="geolocation">Geolocation</h2>
-            </div>
-            <div class="card-section">
-              <table id="geolocation-table" class="table unstriped">
-                <thead>
-                  <tr>
-                    <th>Country Code</th>
-                    <th>Total Hits</th>
-                    <th>Total Visits</th>
-                    <th>IPs</th>
-                  </tr>              
-                </thead>
-                <tbody>
-                  <%# IP, Hits, Visits Size, Country%>
-                  <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
-                    <tr>
-                      <td class="country"><%= k %></td>
-                      <td class="total-hits"><%= v.map { |x| x[1] }.inject(&:+) %></td>
-                      <td class="total-visits"><%= v.map { |x| x[2] }.inject(&:+) %></td>
-                      <td class="ips">
-                        <%= v.map { |x| "<a href=\"https://whatismyipaddress.com/ip/#{x[0]}\">#{x[0]}</a>" }.join(", ") %>
-                      </td>
-                    </tr>
-                  <% end %>
-                </tbody>
-              </table>
-            </div>
-          </article>
-
-          <article class="card">
-            <div class="card-divider">
-              <h2 id="streaks">Streaks</h2>
-            </div>
-            <div class="card-section">
-              <table id="streaks-table" class="table data-table streaks">
-                <thead>
-                  <tr>
-                    <th>IP</th>
-                    <th>
-                      <div class="grid-x grid-margin-x">
-                        <div class="small-2 cell">
-                          Day
-                        </div>
-                        <div class="small-10 cell">
-                          Resources
-                        </div>
-                      </div>
-                    </th>
-                  </tr>
-                </thead>
-                <tbody>
-                  <% data[:streaks].group_by(&:first).each do |ip, date_urls| %>
-                    <tr>
-                      <td class="ip">
-                        <a href="https://whatismyipaddress.com/ip/<%= ip %>"><%= ip %></a>
-                      </td>
-                      <td class="streaks">
-                        <div class="grid-x grid-margin-x">
-                          <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
-                            <div class="small-12 medium-1 cell">
-                              <span class="date"><%= date %></span>
-                            </div>
-                            <div class="small-12 medium-5 cell">
-                              <span class="res-title">HTML:</span>
-                              <% unique_with_count = urls.map { |x| x[2] }.compact.group_by{|e| e}.map{|k, v| [k, v.length]} %>
-                              <ul class="no-bullet">
-                                <% unique_with_count.select { |x| x[0].match /.*\.html?/ }.each do |url| %>
-                                  <li>[<%= url[1] %>] <%= Emitter::escape_javascript url[0] %></li>
-                                <% end %>
-                              </ul>
-                            </div>                              
-                            <div class=" small-12 medium-5 cell">
-                              <span class="res-title">Other Resources:</span>
-                              <ul class="no-bullet">
-                                <% unique_with_count.select { |x| x[0] and ! x[0].match /.*\.html?/ }.each do |url| %>
-                                  <li>[<%= url[1] %>] <%= Emitter::escape_javascript url[0] %></li>
-                                <% end %>
-                              </ul>
-                            </div>
-                          <% end %>
-                        </div>
-                      </td>
-                    </tr>
-                  <% end %>
-                </tbody>
-              </table>
-            </div>
-          </article>
-
           <div class="grid-x grid-margin-x">
             <div class="cell small-12 large-6">
               <article>
-                <h2 id="command-invocation">Command Invocation</h2>
+                <h2 id="<%= Emitter::slugify "Command Invocation" %>">Command Invocation</h2>
 
                 <%= render "command_invocation.html.erb", data: data, options: options %>
               </article>
@@ -354,7 +234,7 @@
 
             <div class="small-12 large-6 cell">
               <article>
-                <h2 id="performance">Performance</h2>
+                <h2 id="<%= Emitter::slugify "Performance" %>">Performance</h2>
 
                 <%=  render "performance.html.erb", data: data %>
               </article>

data/lib/log_sense/templates/apache.txt.erb

@@ -1,30 +1,17 @@
 * Apache Log Analysis
 
->>>>  URLs NOT SANITIZED. DO NOT CONVERT TO HTML <<<<
->>>>    (USE THE HTML EXPORT FUNCTION INSTEAD)   <<<<
+<%= render "warning.txt.erb" %>
 
 ** Summary
 
 <%= render "summary.txt.erb", data: data %>
 
-<% @reports.each do |report| %>
+<% @reports.reject { |x| x[:report] == :html }.each do |report| %>
 ** <%= report[:title] %>
 
 <%= render "output_table.txt.erb", report: report, data: data %>
 <% end %>
 
-** Geolocation
-
-<%=
-ips = data[:ips].group_by { |x| x[4] }.map { |k, v|
-  [k, v.map { |x| x[1] }.inject(&:+), v.map { |x| x[2] }.inject(&:+) ]
-}
-table = Terminal::Table.new headings: ["Country", "Hits", "Total Visits"], rows: ips
-table.style = { border_i: "|" }
-(1..2).map { |i| table.align_column(i, :right) }
-table
-%>
-
 ** Command Invocation
 
 <%= render 'command_invocation.txt.erb', data: data %>

data/lib/log_sense/templates/rails.html.erb

@@ -1,7 +1,9 @@
 <!doctype html>
 <html class="no-js" lang="en">
   <head>
-    <title><%= options[:title] || "Log Sense: #{data[:log_file]}" %></title>
+    <title>
+      <%= options[:title] || "Log Sense: #{data[:filenames].empty? ? "stdin" : data[:filenames].join(", ")}" %>
+    </title>
 
     <meta charset="utf-8" />
     <meta http-equiv="x-ua-compatible" content="ie=edge">
@@ -150,36 +152,8 @@
   <body>
     <div class="off-canvas-wrapper">
       <div class="off-canvas position-left" id="offCanvas" data-off-canvas>
-        <nav>
-          <h2>Navigation</h2>
-          <ul class="no-bullet">
-            <% [
-              "Summary",
-              "Log Structure",
-              "Daily Distribution",
-              "Time Distribution",
-              "Statuses",
-              "Rails Performance",
-              "Fatal Events",
-              "Internal Server Errrors",
-              "Errors",
-              "IPs",
-              "Command Invocation",
-              "Performance"
-            ].each do |item| %>
-              <li class="nav-item">
-                <a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
-              </li>
-            <% end %>
-          </ul>
-
-          <p>
-            Generated by
-            <a href="https://github.com/avillafiorita/log_sense">LogSense</a> <br />
-            on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
-            <a href='https://db-ip.com'>IP Geolocation by DB-IP</a>
-          </p>
-        </nav>
+        <%= render "navigation.html.erb",
+                   menus: Emitter::rails_report_specification.map { |x| x[:title] } %>
       </div>
       <div class="off-canvas-content grid-container grid-x fluid" data-off-canvas-content>
         <div data-sticky-container>
@@ -191,14 +165,14 @@
         </div>
 
         <section class="main-section">
-          <h1><%= options[:title] || "Log Sense: #{data[:log_file]}" %></h1>
+          <h1><%= options[:title] || "Log Sense Rails Log Report" %></h1>
 
-          <p><b>Input File:</b>  <%= (data[:log_file] || "stdin") %></p>
+          <p><b>Input File(s):</b>  <%= data[:filenames].empty? ? "stdin" : data[:filenames].join(", ") %></p>
 
           <div class="grid-x grid-margin-x">
             <article class="card small-12 large-6 cell">
               <div class="card-divider">
-                <h2 id="summary">Summary</h2>
+                <h2 id="<%= Emitter::slugify "Summary" %>">Summary</h2>
               </div>
               <div class="card-section">
                 <%= render "summary.html.erb", data: data %>
@@ -207,7 +181,7 @@
 
             <article class="card cell small-12 large-6">
               <div class="card-divider">
-                <h2 id="log-structure">Log Structure</h2>
+                <h2 id="<%= Emitter::slugify "Log Structure" %>">Log Structure</h2>
               </div>
               <div class="card-section">
                 <%= render "log_structure.html.erb", data: data %>
@@ -249,7 +223,9 @@
           <div class="grid-x grid-margin-x">
             <div class="cell small-12 large-6">
               <article>
-                <h2 id="command-invocation">Command Invocation</h2>
+                <h2 id="<%= Emitter::slugify "Command Invocation" %>">
+                  Command Invocation
+                </h2>
 
                 <%= render "command_invocation.html.erb", data: data, options: options %>
               </article>
@@ -257,7 +233,7 @@
 
             <div class="small-12 large-6 cell">
               <article>
-                <h2 id="performance"> Performance</h2>
+                <h2 id="<%= Emitter::slugify "Performance" %>"> Performance</h2>
 
                 <%=  render "performance.html.erb", data: data %>
               </article>

data/lib/log_sense/templates/rails.txt.erb

@@ -1,13 +1,12 @@
 * Rails Log Analysis
 
->>>>  URLs NOT SANITIZED. DO NOT CONVERT TO HTML <<<<
->>>>    (USE THE HTML EXPORT FUNCTION INSTEAD)   <<<<
+<%= render "warning.txt.erb" %>
 
 ** Summary
 
 <%= render "summary.txt.erb", data: data %>
 
-<% @reports.each do |report| %>
+<% @reports.reject { |x| x[:report] == :html }.each do |report| %>
 ** <%= report[:title] %>
 
 <%= render "output_table.txt.erb", report: report, data: data %>
@@ -15,9 +14,9 @@
 
 ** Command Invocation
 
-<%= render "command_invocation.txt.erb", data: data %>
+<%= render 'command_invocation.txt.erb', data: data %>
 
-** Log Sense Performance
+** Performance
 
-<%= render "performance.txt.erb", data: data %>
+<%= render 'performance.txt.erb', data: data %>
 

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "1.4.1"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Adolfo Fibrillation
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-07 00:00:00.000000000 Z
+date: 2022-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser
@@ -141,6 +141,7 @@ files:
 - lib/log_sense/templates/_command_invocation.html.erb
 - lib/log_sense/templates/_command_invocation.txt.erb
 - lib/log_sense/templates/_log_structure.html.erb
+- lib/log_sense/templates/_navigation.html.erb
 - lib/log_sense/templates/_output_table.html.erb
 - lib/log_sense/templates/_output_table.txt.erb
 - lib/log_sense/templates/_performance.html.erb
@@ -148,6 +149,7 @@ files:
 - lib/log_sense/templates/_report_data.html.erb
 - lib/log_sense/templates/_summary.html.erb
 - lib/log_sense/templates/_summary.txt.erb
+- lib/log_sense/templates/_warning.txt.erb
 - lib/log_sense/templates/apache.html.erb
 - lib/log_sense/templates/apache.txt.erb
 - lib/log_sense/templates/rails.html.erb