log_sense 1.5.1 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f20a0d2041df1f2414bd0015fff27764c28a89dfd2d45fdb275141638bc5784
-  data.tar.gz: 14426b7383fe9b0077c2852f177545384a369f35489aa5d95372c20cbe19732d
+  metadata.gz: 5f37b2247014af9bccfb8bdd205b54eb51cbef35495904444765648a96e0b9ac
+  data.tar.gz: a9cd03afb6770bf854791b8ec93e87d09532828d894b28f5dde670f1af1b1b1d
 SHA512:
-  metadata.gz: 9defcff35a5b3802d7d1a7db596a30aa01e28d3b9d784619de5f61713a587e427ccce76fcfcd478e946a0822af8d0822b23bad6392c2a71690ccd07250d5cfb7
-  data.tar.gz: 8b9143feb4f7508de9b7f60eddacafc4713c064f8bda21ed9ae92de364aa5e94fed36ff6d12f2b2aa108337d92c25868bce21d172397f48ff3c41ed93dc9a2b5
+  metadata.gz: 62981216e38b92e1c3ea227726dccd592c441de32519a4df6f39bac127f55da32e82b4a1a14897b09b7032c7b3f9506a14e80e49dc43bfe8c9d86bfaa340da82
+  data.tar.gz: e5e6180008a12561d688668cffb2ef3d885d7733cf877aafaed0397b9fa0e91dd12a2998b19006dad4fe6b202bd203a00757f86d5d37efee777dbc7be43e5ab1

data/CHANGELOG.org

@@ -2,6 +2,29 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.5.2
+
+- [User] Updated DB-IP country file.
+- [User] Added reports "Missed Pages by IP" and "Missed Resources by
+  IP".  It can help pinpoint attack sources.
+- [User] Added report "Combined Platform", which puts together
+  Browser, OS, and IP.
+- [User] Summary now shows total size transferred.
+- [User] Added link to DB-IP page for IPs in some tables.
+- [User] Added count of IPs by Country.
+- [User] Improved textual report: values in cells holding multiple
+  values (e.g. IPs) are now shown in distinct lines in the cell. A new
+  option -r limits the number of lines shown per cell.
+- [Default] The number of rows initially shown in HTML reports is now 25.
+- [Default] Default for number of entries in textual report is now
+  100 (used to be 900).
+- [Fixed] The size column in HTML reports is now sorted numerically.
+- [Code] Improved performances of DataTable rendering, using the
+  dataRender flag.
+- [Code] Use trim_mode in ERB to avoid empty lines in HTML output.
+- [Code] Moved to the debug gem.
+- [Gem] Updated email and author's name.
+
 * 1.5.1
 
 - [User] Option --input-files allows to specify input files
@@ -16,7 +39,7 @@
 - [User] Present Unique Visits / day as integer
 - [User] Added Country and Streaks report for rails
 - [User] Changed Streak report in Apache
-- [Gem] Updated DBIP
+- [Gem] Updated DB-IP
 - [Gem] Updated Bundle  
 - [Code] Refactored all reports, so that they are specified
   in the same way  

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.4.2)
+    log_sense (1.5.2)
       browser
       ipaddr
       iso_country_codes
@@ -12,21 +12,28 @@ GEM
   remote: https://rubygems.org/
   specs:
     browser (5.3.1)
-    byebug (11.1.3)
+    debug (1.6.2)
+      irb (>= 1.3.6)
+      reline (>= 0.3.1)
+    io-console (0.5.11)
     ipaddr (1.2.4)
+    irb (1.4.1)
+      reline (>= 0.3.0)
     iso_country_codes (0.7.8)
     minitest (5.15.0)
     rake (12.3.3)
-    sqlite3 (1.4.2)
+    reline (0.3.1)
+      io-console (~> 0.5)
+    sqlite3 (1.4.4)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.1.0)
+    unicode-display_width (2.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  byebug
+  debug
   log_sense!
   minitest
   rake (~> 12.0)

data/LICENSE.txt

@@ -1,5 +1,4 @@
-The MIT License (MIT)
-
+The source code is distributed under the terms of the MIT License (MIT)
 Copyright (c) 2021 Shair.Tech
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -19,3 +18,11 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+
+==============================================================================
+LogSense uses data from The free DB-IP Lite database for geolocation purposes.
+
+The DB-IP Lite Database is licensed under a Creative Commons Attribution 4.0
+International License.
+
+For more information: https://db-ip.com 

data/README.org

@@ -14,10 +14,12 @@ LogSense reports the following data:
 - Visitors, hits, unique visitors, bandwidth used
 - Most accessed HTML pages
 - Most accessed resources  
+- Missed resources (also by IP) which helps highlight
+  potential attacks
 - Response statuses
 - Referers
 - OS, browsers, and devices
-- IP Country location, thanks to the DPIP lite country DB
+- IP Country location, thanks to the DP-IP lite country DB
 - Streaks: resources accessed by a given IP over time
 - Performance of Rails requests
  
@@ -81,32 +83,33 @@ generated files are then made available on a private area on the web.
       -o, --output-file=OUTPUT_FILE    Output file
       -b, --begin=DATE                 Consider entries after or on DATE
       -e, --end=DATE                   Consider entries before or on DATE
-      -l, --limit=N                    Limit to the N most requested resources (defaults to 900)
-      -w, --width=WIDTH                Maximum width of URL and description columns in text reports
+      -l, --limit=N                    Limit to the N most requested resources (defaults to 100)
+      -w, --width=WIDTH                Maximum width of long columns in textual reports
+      -r, --rows=ROWS                  Maximum number of rows for columns with multiple entries in textual reports
       -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
       -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
           --verbose                    Inform about progress (prints to STDERR)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.5.1
+  This is version 1.5.2
 
   Output formats
-  rails parsing can produce the following outputs:
-    - sqlite
-    - txt
-    - html
   apache parsing can produce the following outputs:
     - sqlite
+    - html
     - txt
+  rails parsing can produce the following outputs:
+    - sqlite
     - html
+    - txt
   #+end_example
 
 Examples:
 
 #+begin_example sh
 log_sense -f apache -i access.log -t txt > access-data.txt
-log_sense -f rails -i production.log -t html -o performance.txt
+log_sense -f rails -i production.log -t html -o performance.html
 #+end_example
 
 
@@ -138,7 +141,7 @@ the known bugs.)
 
 * License
 
-Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
+Source code distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
 
 Geolocation is made possible by the DB-IP.com IP to City database,
 released under a CC license.

data/lib/log_sense/apache_data_cruncher.rb

@@ -101,6 +101,9 @@ module LogSense
       @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
       @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
+      @missed_pages_by_ip = db.execute "SELECT ip, path, status from LogLine where #{bad_statuses} and #{html_page} and #{filter} limit #{options[:limit]}"
+      @missed_resources_by_ip = db.execute "SELECT ip, path, status from LogLine where #{bad_statuses} and #{filter} limit #{options[:limit]}"
+
       @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
 
       @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"
@@ -113,6 +116,9 @@ module LogSense
 
       @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
       @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
+
+      @combined_platforms = db.execute "SELECT browser, platform, ip, count(datetime), #{human_readable_size} from LogLine where #{filter} group by browser, platform, ip order by count(datetime) desc limit #{options[:limit]}"
+
       @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
       
       @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"

data/lib/log_sense/emitter.rb

@@ -4,10 +4,24 @@ require 'json'
 require 'erb'
 require 'ostruct'
 module LogSense
+  WORDS_SEPARATOR = ' · '
+
   #
   # Emit Data
   #
   module Emitter
+    def self.human_readable_size(size)
+      if size < 1024
+        "%d B" % size
+      elsif size < 1024 * 1024
+        "%.2f KB" % (size.to_f / 1024)
+      elsif size < 1024 * 1024 * 1024
+        "%.2f MB" % (size.to_f / (1024 * 1024))
+      else
+        "%.2f GB" % (size.to_f / (1024 * 1024 * 1024))
+      end
+    end
+    
     def self.emit(data = {}, options = {})
       @input_format = options[:input_format] || 'apache'
       @output_format = options[:output_format] || 'html'
@@ -20,7 +34,7 @@ module LogSense
       # determine the main template to read
       @template = File.join(File.dirname(__FILE__), 'templates', "#{@input_format}.#{@output_format}.erb")
       erb_template = File.read @template
-      output = ERB.new(erb_template).result(binding)
+      output = ERB.new(erb_template, trim_mode: "-").result(binding)
 
       if options[:output_file]
         file = File.open options[:output_file], 'w'
@@ -31,10 +45,14 @@ module LogSense
       end
     end
 
+    #
+    # This is used in templates
+    #
     def self.render(template, vars = {})
       @template = File.join(File.dirname(__FILE__), 'templates', "_#{template}")
       erb_template = File.read @template
-      ERB.new(erb_template).result(OpenStruct.new(vars).instance_eval { binding })
+      ERB.new(erb_template, trim_mode: "-")
+        .result(OpenStruct.new(vars).instance_eval { binding })
     end
 
     def self.escape_javascript(string)
@@ -65,23 +83,37 @@ module LogSense
       [Integer, Float].include?(klass) ? value : escape_javascript(value || '')
     end
 
-    # limit width of special columns, that is, URL, Path, and Description
+    # limit width of special columns, that is, those in keywords
     # - data: array of arrays
     # - heading: array with column names
     # - width width to set
-    def self.shorten(data, heading, width)
-      # indexes of columns which have to be shortened
-      keywords = %w[URL Referers Description Path]
-      to_shorten = keywords.map { |x| heading.index x }.compact
+    def self.shorten(data, heading, width, inner_rows)
+      # columns which need to be shortened
+      keywords = ["URL", "Referers", "Description", "Path", "Paths", "IP List"]
+      # indexes of columns which have to be shortened (= index in array)
+      to_shorten = keywords.map { |idx| heading.index idx }.compact
 
-      if width.nil? || to_shorten.empty? || data[0].nil?
+      if data[0].nil? || width.nil? || to_shorten.empty?
         data
       else
+        # how many columns do we have?
         table_columns = data[0].size
         data.map do |x|
+          # we iterate over all columns, because we want to return a table
+          # with the same structure of the input table
           (0..table_columns - 1).each.map do |col|
-            should_shorten = x[col] && x[col].size > width - 3 && to_shorten.include?(col)
-            should_shorten ? "#{x[col][0..(width - 3)]}..." : x[col]
+            # split cell into (internal) rows, if necessary
+            content_in_rows = x[col].to_s.split WORDS_SEPARATOR
+            # remove excess rows, shorten each string and return what's left
+            # single cells are returned as they are
+            rows_limit = inner_rows || content_in_rows.size
+            content = content_in_rows[0..(rows_limit - 1)].map { |x|
+              if x.size > width && to_shorten.include?(col)
+                "#{x[0..(width - 3)]}..."
+              else
+                x
+              end
+            }.join("\n")
           end
         end
       end
@@ -119,7 +151,7 @@ module LogSense
                        },
                        {
                          'mark': {
-                                   'type': 'text',
+                                  'type': 'text',
                                   'color': '#3E5772',
                                   'align': 'middle',
                                   'baseline': 'top',
@@ -127,14 +159,14 @@ module LogSense
                                   'yOffset': -15
                                  },
                         'encoding': {
-                                      'text': {'field': 'Hits', 'type': 'quantitative'},
+                                     'text': {'field': 'Hits', 'type': 'quantitative'},
                                      'y': {'field': 'Hits', 'type': 'quantitative'}
                                     }
                        },
 
                        {
                          'mark': {
-                                   'type': 'line',
+                                  'type': 'line',
                                   'color': '#A52A2A',
                                   'point': {
                                              'color': '#A52A2A',
@@ -149,7 +181,7 @@ module LogSense
 
                        {
                          'mark': {
-                                   'type': 'text',
+                                  'type': 'text',
                                   'color': '#A52A2A',
                                   'align': 'middle',
                                   'baseline': 'top',
@@ -203,28 +235,58 @@ module LogSense
           header: %w[Path Hits Visits Size Status],
           column_alignment: %i[left right right right right],
           rows: data[:most_requested_pages],
-          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
+          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 }, { width: \'15%\', targets: [1, 2, 3, 4] }], dataRender: true'
         },
         {
           title: '20_ and 30_ on other resources',
           header: %w[Path Hits Visits Size Status],
           column_alignment: %i[left right right right right],
           rows: data[:most_requested_resources],
-          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
+          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 }, { width: \'15%\', targets: [1, 2, 3, 4] }], dataRender: true'
         },
         {
           title: '40_ and 50_x on HTML pages',
           header: %w[Path Hits Visits Status],
           column_alignment: %i[left right right right],
           rows: data[:missed_pages],
-          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
+          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 }, { width: \'20%\', targets: [1, 2, 3] }], dataRender: true'
         },
         {
           title: '40_ and 50_ on other resources',
           header: %w[Path Hits Visits Status],
           column_alignment: %i[left right right right],
           rows: data[:missed_resources],
-          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
+          datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 }, { width: \'20%\', targets: [1, 2, 3] }], dataRender: true'
+        },
+        {
+          title: '40_ and 50_x on HTML pages by IP',
+          header: %w[IP Hits Paths],
+          column_alignment: %i[left right left],
+          # Value is something along the line of:
+          # [["66.249.79.93", "/adolfo/notes/calendar/2014/11/16.html", "404"],
+          #  ["66.249.79.93", "/adolfo/website-specification/generate-xml-sitemap.org.html", "404"]]
+          rows: data[:missed_pages_by_ip]&.group_by { |x| x[0] }&.map { |k, v|
+            [
+              k,
+              v.size,
+              v.map { |x| x[1] }.join(WORDS_SEPARATOR)
+            ]
+          }&.sort { |x, y| y[1] <=> x[1] }
+        },
+        {
+          title: '40_ and 50_ on other resources by IP',
+          header: %w[IP Hits Paths],
+          column_alignment: %i[left right left],
+          # Value is something along the line of:
+          # [["66.249.79.93", "/adolfo/notes/calendar/2014/11/16.html", "404"],
+          #  ["66.249.79.93", "/adolfo/website-specification/generate-xml-sitemap.org.html", "404"]]
+          rows: data[:missed_resources_by_ip]&.group_by { |x| x[0] }&.map { |k, v|
+            [
+              k,
+              v.size,
+              v.map { |x| x[1] }.join(WORDS_SEPARATOR)
+            ]
+          }&.sort { |x, y| y[1] <=> x[1] }
         },
         {
           title: 'Statuses',
@@ -233,10 +295,10 @@ module LogSense
           rows: data[:statuses],
           vega_spec: {
             'mark': 'bar',
-                      'encoding': {
-                                    'x': {'field': 'Status', 'type': 'nominal'},
-                                   'y': {'field': 'Count', 'type': 'quantitative'}
-                                  }
+            'encoding': {
+               'x': {'field': 'Status', 'type': 'nominal'},
+               'y': {'field': 'Count', 'type': 'quantitative'}
+            }
           }
         },
         {
@@ -246,27 +308,27 @@ module LogSense
           rows: data[:statuses_by_day],
           vega_spec: {
             'transform': [ {'fold': ['S_2xx', 'S_3xx', 'S_4xx' ] }],
-                      'mark': 'bar',
-                      'encoding': {
-                                    'x': {
-                                           'field': 'Date',
-                                          'type': 'ordinal',
-                                          'timeUnit': 'day', 
-                                         },
-                                   'y': {
-                                          'aggregate': 'sum',
-                                         'field': 'value',
-                                         'type': 'quantitative'
-                                        },
-                                   'color': {
-                                              'field': 'key',
-                                             'type': 'nominal',
-                                             'scale': {
-                                                        'domain': ['S_2xx', 'S_3xx', 'S_4xx'],
-                                                       'range': ['#228b22', '#ff8c00', '#a52a2a']
-                                                      },
-                                            }
-                                  }
+            'mark': 'bar',
+            'encoding': {
+              'x': { 
+                'field': 'Date',
+                'type': 'ordinal',
+                'timeUnit': 'day', 
+              },
+              'y': {
+                'aggregate': 'sum',
+                'field': 'value',
+                'type': 'quantitative'
+              },
+              'color': {
+                'field': 'key',
+                'type': 'nominal',
+                'scale': {
+                  'domain': ['S_2xx', 'S_3xx', 'S_4xx'],
+                  'range': ['#228b22', '#ff8c00', '#a52a2a']
+                },
+              }
+            }
           }
         },
         {
@@ -325,27 +387,36 @@ module LogSense
         },
         {
           title: 'IPs',
-          header: %w[IPs Hits Visits Size Country],
+          header: %w[IP Hits Visits Size Country],
           column_alignment: %i[left right right right left],
           rows: data[:ips]
         },
         {
           title: 'Countries',
-          header: %w[Country Hits Visits IPs],
-          column_alignment: %i[left right right left],
-          rows: data[:countries]&.map do |k, v|
+          header: ["Country", "Hits", "Visits", "IPs", "IP List"],
+          column_alignment: %i[left right right right left],
+          rows: data[:countries]&.map { |k, v|
             [
               k,
               v.map { |x| x[1] }.inject(&:+),
               v.map { |x| x[2] }.inject(&:+),
-              v.map { |x| x[0] }.join(' ')
+              v.map { |x| x[0] }.size,
+              v.map { |x| x[0] }.join(WORDS_SEPARATOR)
             ]
-          end
+          }&.sort { |x, y| y[3] <=> x[3] }
+        },
+        {
+          title: 'Combined Platform Data',
+          header: %w[ Browser OS IP Hits Size],
+          column_alignment: %i[left left left right right],
+          col: 'small-12 cell',
+          rows: data[:combined_platforms],
         },
         {
           title: 'Referers',
           header: %w[Referers Hits Visits Size],
           column_alignment: %i[left right right right],
+          datatable_options: 'columnDefs: [{ width: \'50%\', targets: 0 } ], dataRender: true',
           rows: data[:referers],
           col: 'small-12 cell'
         },
@@ -354,14 +425,15 @@ module LogSense
           report: :html,
           header: ['IP', 'Date', 'Total HTML', 'Total Other', 'HTML', 'Other'],
           column_alignment: %i[left left right right left left],
+          datatable_options: 'columnDefs: [{ width: \'30%\', targets: [4, 5] }, { width: \'10%\', targets: [0, 1, 2, 3]} ], dataRender: true',
           rows: data[:streaks]&.group_by { |x| [x[0], x[1]] }&.map do |k, v|
             [
               k[0],
               k[1],
               v.map { |x| x[2] }.compact.select { |x| x.match(/\.html?$/) }.size,
               v.map { |x| x[2] }.compact.reject { |x| x.match(/\.html?$/) }.size,
-              v.map { |x| x[2] }.compact.select { |x| x.match(/\.html?$/) }.join(' · '),
-              v.map { |x| x[2] }.compact.reject { |x| x.match(/\.html?$/) }.join(' · ')
+              v.map { |x| x[2] }.compact.select { |x| x.match(/\.html?$/) }.join(WORDS_SEPARATOR),
+              v.map { |x| x[2] }.compact.reject { |x| x.match(/\.html?$/) }.join(WORDS_SEPARATOR)
             ]
           end,
           col: 'small-12 cell'
@@ -528,13 +600,13 @@ module LogSense
           title: 'Countries',
           header: %w[Country Hits IPs],
           column_alignment: %i[left right left],
-          rows: data[:countries]&.map do |k, v|
+          rows: data[:countries]&.map { |k, v|
             [
               k,
               v.map { |x| x[1] }.inject(&:+),
-              v.map { |x| x[0] }.join(' · ')
+              v.map { |x| x[0] }.join(WORDS_SEPARATOR)
             ]
-          end
+          }&.sort { |x, y| x[0] <=> y[0] }
         },
         {
           title: 'Streaks',
@@ -546,7 +618,7 @@ module LogSense
               k[0],
               k[1],
               v.size,
-              v.map { |x| x[2] }.join(' · ')
+              v.map { |x| x[2] }.join(WORDS_SEPARATOR)
             ]
           end,
           col: 'small-12 cell'

data/lib/log_sense/options_parser.rb

@@ -8,7 +8,7 @@ module LogSense
     # parse command line options
     #
     def self.parse(options)
-      limit = 900
+      limit = 100
       args = {} 
 
       opt_parser = OptionParser.new do |opts|
@@ -46,10 +46,14 @@ module LogSense
           args[:limit] = n
         end
 
-        opts.on('-wWIDTH', '--width=WIDTH', Integer, 'Maximum width of URL and description columns in text reports') do |n|
+        opts.on('-wWIDTH', '--width=WIDTH', Integer, 'Maximum width of long columns in textual reports') do |n|
           args[:width] = n
         end
 
+        opts.on('-rROWS', '--rows=ROWS', Integer, 'Maximum number of rows for columns with multiple entries in textual reports') do |n|
+          args[:inner_rows] = n
+        end
+
         opts.on('-cPOLICY', '--crawlers=POLICY', String, 'Decide what to do with crawlers (applies to Apache Logs)') do |n|
           case n
           when 'only'

data/lib/log_sense/templates/_log_structure.html.erb

@@ -8,13 +8,16 @@
     <span class="stats-list-label">To</span>
   </li>
   <li class="stats-list-positive">
-    <%= data[:total_days] %> <span class="stats-list-label">Days in Log</span>
+    <%= data[:total_days] %>
+    <span class="stats-list-label">Days in Log</span>
   </li>
   <li class="stats-list-negative">
-    <%= data[:log_size] %> <span class="stats-list-label">Total Entries</span>
+    <%= data[:log_size] %>
+    <span class="stats-list-label">Total Entries</span>
   </li>
   <li class="stats-list-negative">
-    <%= data[:selfpolls_size] %> <span class="stats-list-label">Self Polls</span>
+    <%= data[:selfpolls_size] %>
+    <span class="stats-list-label">Self Polls</span>
   </li>
   <li class="stats-list-negative">
     <td><%= data[:crawlers_size] %></td>

data/lib/log_sense/templates/_output_table.html.erb

@@ -13,11 +13,59 @@
  $(document).ready(function(){
    $('#table-<%= index %>').dataTable({
      data: data_<%= index %>,
+     pageLength: 25,
      <%= report[:datatable_options] + "," if report[:datatable_options] %>
      columns: [
        <% report[:header].each do |header| %>
+       <% if header == "Size" -%>
+       {
+         data: 'Size',
+         className: '<%= Emitter::slugify("Size") %>',
+         render: function(data, type, row) {
+           // If display or filter data is requested, format the date
+           if ( type === 'display' || type === 'filter' ) {
+            return data;
+           }
+           // Otherwise the data type requested (`type`) is type detection or
+           // sorting data, for which we want to use an integer value
+           value = data.split(/(\s+)/);
+           size = parseInt(value[0])
+           switch(value[2]) {
+             case 'B':
+               multiplier = 1
+               break;
+             case 'KB':
+               multiplier = 1024
+               break;
+             case 'MB':
+               multiplier = 1024 * 1024
+               break;
+             case 'GB':
+               multiplier = 1024 * 1024 * 1024
+               break;
+             default:
+               multiplier = 1
+           } 
+           return size * multiplier;
+         }
+       },
+       <% elsif header == "IP" -%>
+       {
+         data: 'IP',
+         className: '<%= Emitter::slugify("IP") %>',
+         render: function(data, type, row) {
+           // If display or filter data is requested, format the date
+           if ( type === 'display' || type === 'filter' ) {
+             return "<a target=\"_blank\" href=\"https://db-ip.com/" + data + "\">" + data + "</a>";
+           }
+           // For any other purpose return data
+           return data;
+         }
+       },
+       <% else -%>
        { data: '<%= header %>', className: '<%= Emitter::slugify(header) %>' },
-       <% end %>
+       <% end -%>
+       <% end -%>
      ]
    });
  });

data/lib/log_sense/templates/_output_table.txt.erb

@@ -1,6 +1,8 @@
 <%=
 # shortens long URLs and long descriptions
-shortened = Emitter::shorten(report[:rows], report[:header], data[:width])
+shortened = Emitter::shorten(
+  report[:rows], report[:header], data[:width], data[:inner_rows]
+)
 
 # build and style the table
 table = Terminal::Table.new headings: report[:header], rows: shortened

data/lib/log_sense/templates/_report_data.html.erb

@@ -1,12 +1,12 @@
 <script>
  /* this is used both by Vega and DataTable for <%= report[:title] %>*/
  data_<%= index %> = [
-   <% report[:rows].each do |row| %>
+   <% report[:rows].each do |row| -%>
    {
-     <% report[:header].each_with_index do |h, i| %>
+     <% report[:header].each_with_index do |h, i| -%>
      "<%= h %>": "<%= Emitter::process row[i] %>",
-     <% end %>
+     <% end -%>
    },
-   <% end %>
+   <% end -%>
  ]
 </script>

data/lib/log_sense/templates/_summary.html.erb

@@ -8,13 +8,20 @@
     <span class="stats-list-label">To</span>
   </li>
   <li class="stats-list-positive">
-    <%= data[:total_days_in_analysis] %> <span class="stats-list-label">Days</span>
+    <%= data[:total_days_in_analysis] %>
+    <span class="stats-list-label">Days</span>
+  </li>
+  <li class="stats-list-positive">
+    <%= data[:total_size] %>
+    <span class="stats-list-label">Data Transferred</span>
   </li>
   <li class="stats-list-negative">
-    <%= data[:total_hits] %> <span class="stats-list-label">Hits</span>
+    <%= data[:total_hits] %>
+    <span class="stats-list-label">Hits</span>
   </li>
   <li class="stats-list-negative">
-    <%= data[:total_unique_visits] %> <span class="stats-list-label">Unique Visits</span>
+    <%= data[:total_unique_visits] %>
+    <span class="stats-list-label">Unique Visits</span>
   </li>
   <li class="stats-list-negative">
     <% days = data[:last_day_in_analysis] - data[:first_day_in_analysis] %>

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "1.5.1"
+  VERSION = "1.5.2"
 end

data/log_sense.gemspec

@@ -1,38 +1,38 @@
-require_relative 'lib/log_sense/version'
+require_relative "lib/log_sense/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = 'log_sense'
+  spec.name          = "log_sense"
   spec.version       = LogSense::VERSION
-  spec.authors       = ['Adolfo Fibrillation']
-  spec.email         = ['adolfo@shair.tech']
+  spec.authors       = ["Adolfo Villafiorita"]
+  spec.email         = ["adolfo@shair.tech"]
 
   spec.summary       = %q{Generate analytics for Apache and Rails log file.}
   spec.description   = %q{Generate analytics in HTML, txt, and SQLite format for Apache and Rails log files.}
-  spec.homepage      = 'https://github.com/shair-tech/log_sense/log_sense'
-  spec.license       = 'MIT'
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.6.9')
+  spec.homepage      = "https://github.com/shair-tech/log_sense/log_sense"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.6.9")
 
-  spec.metadata['allowed_push_host'] = 'https://rubygems.org/'
+  spec.metadata["allowed_push_host"] = "https://rubygems.org/"
 
-  spec.metadata['homepage_uri'] = spec.homepage
-  spec.metadata['source_code_uri'] = 'https://github.com/shair-tech/log_sense/log_sense'
-  spec.metadata['changelog_uri'] = 'https://github.com/shair-tech/log_sense/blob/main/CHANGELOG.org'
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/shair-tech/log_sense/"
+  spec.metadata["changelog_uri"] = "https://github.com/shair-tech/log_sense/blob/main/CHANGELOG.org"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path("..", __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = 'exe'
+  spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ['lib']
+  spec.require_paths = ["lib"]
 
-  spec.add_dependency 'browser'
-  spec.add_dependency 'ipaddr'
-  spec.add_dependency 'iso_country_codes'
-  spec.add_dependency 'sqlite3'
-  spec.add_dependency 'terminal-table'
+  spec.add_dependency "browser"
+  spec.add_dependency "ipaddr"
+  spec.add_dependency "iso_country_codes"
+  spec.add_dependency "sqlite3"
+  spec.add_dependency "terminal-table"
 
-  spec.add_development_dependency 'byebug'
-  spec.add_development_dependency 'minitest'
+  spec.add_development_dependency "debug"
+  spec.add_development_dependency "minitest"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.5.2
 platform: ruby
 authors:
-- Adolfo Fibrillation
+- Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-09 00:00:00.000000000 Z
+date: 2022-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -157,7 +157,6 @@ files:
 - lib/log_sense/templates/rails.txt.erb
 - lib/log_sense/version.rb
 - log_sense.gemspec
-- sample_logs/empty_log.log
 - sample_logs/safety-critical_org.log
 - sample_logs/spmbook_com.log
 homepage: https://github.com/shair-tech/log_sense/log_sense
@@ -166,7 +165,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org/
   homepage_uri: https://github.com/shair-tech/log_sense/log_sense
-  source_code_uri: https://github.com/shair-tech/log_sense/log_sense
+  source_code_uri: https://github.com/shair-tech/log_sense/
   changelog_uri: https://github.com/shair-tech/log_sense/blob/main/CHANGELOG.org
 post_install_message:
 rdoc_options: []
@@ -183,7 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Generate analytics for Apache and Rails log file.