log_sense 1.4.1 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ef1d35932ba8c7fe6e636f6cb507ea99fd6b6026170d62207dfa2606d62bbcb
-  data.tar.gz: 725dc6e51ace6c9cbd366e3888387ef8f246327c9e1036c66d2eb6351b1f0791
+  metadata.gz: 5f37b2247014af9bccfb8bdd205b54eb51cbef35495904444765648a96e0b9ac
+  data.tar.gz: a9cd03afb6770bf854791b8ec93e87d09532828d894b28f5dde670f1af1b1b1d
 SHA512:
-  metadata.gz: 592e80cd56f4740cc4003b494c9da960de228025377d630d070a08a495cffac7fd41850d469189510def48f9940700251bd84a141def9b1b15b32fbed967261f
-  data.tar.gz: 1d802e95fd58c66c4904843478c5ccc8ffc1ce43e87dd199e5b21efc89bcff436b227c646d2e5a386bee22a3d1c67c0341c25fcb4058e48526560d05984f0148
+  metadata.gz: 62981216e38b92e1c3ea227726dccd592c441de32519a4df6f39bac127f55da32e82b4a1a14897b09b7032c7b3f9506a14e80e49dc43bfe8c9d86bfaa340da82
+  data.tar.gz: e5e6180008a12561d688668cffb2ef3d885d7733cf877aafaed0397b9fa0e91dd12a2998b19006dad4fe6b202bd203a00757f86d5d37efee777dbc7be43e5ab1

data/CHANGELOG.org

@@ -2,6 +2,51 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.5.2
+
+- [User] Updated DB-IP country file.
+- [User] Added reports "Missed Pages by IP" and "Missed Resources by
+  IP".  It can help pinpoint attack sources.
+- [User] Added report "Combined Platform", which puts together
+  Browser, OS, and IP.
+- [User] Summary now shows total size transferred.
+- [User] Added link to DB-IP page for IPs in some tables.
+- [User] Added count of IPs by Country.
+- [User] Improved textual report: values in cells holding multiple
+  values (e.g. IPs) are now shown in distinct lines in the cell. A new
+  option -r limits the number of lines shown per cell.
+- [Default] The number of rows initially shown in HTML reports is now 25.
+- [Default] Default for number of entries in textual report is now
+  100 (used to be 900).
+- [Fixed] The size column in HTML reports is now sorted numerically.
+- [Code] Improved performances of DataTable rendering, using the
+  dataRender flag.
+- [Code] Use trim_mode in ERB to avoid empty lines in HTML output.
+- [Code] Moved to the debug gem.
+- [Gem] Updated email and author's name.
+
+* 1.5.1
+
+- [User] Option --input-files allows to specify input files
+  in addition to passing filenames to the command line
+- [User] Minor changes to the layout of HTML reports
+- [User] Add version number in reports
+- [Fixed] Duplicated entries in navigation
+- [Code] Updated and added minitest(s)
+
+* 1.5.0
+
+- [User] Present Unique Visits / day as integer
+- [User] Added Country and Streaks report for rails
+- [User] Changed Streak report in Apache
+- [Gem] Updated DB-IP
+- [Gem] Updated Bundle  
+- [Code] Refactored all reports, so that they are specified
+  in the same way  
+- [Code] Refactor warning message in textual reports
+- [Code] Build HTML menu for report specification
+- [Code] Various refactoring passes on the code
+
 * 1.4.1
 
 - [User] New textual report for Apache

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.3.1)
+    log_sense (1.5.2)
       browser
       ipaddr
       iso_country_codes
@@ -12,24 +12,31 @@ GEM
   remote: https://rubygems.org/
   specs:
     browser (5.3.1)
-    byebug (11.1.3)
-    ipaddr (1.2.3)
+    debug (1.6.2)
+      irb (>= 1.3.6)
+      reline (>= 0.3.1)
+    io-console (0.5.11)
+    ipaddr (1.2.4)
+    irb (1.4.1)
+      reline (>= 0.3.0)
     iso_country_codes (0.7.8)
-    minitest (5.14.4)
+    minitest (5.15.0)
     rake (12.3.3)
-    sqlite3 (1.4.2)
+    reline (0.3.1)
+      io-console (~> 0.5)
+    sqlite3 (1.4.4)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.1.0)
+    unicode-display_width (2.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  byebug
+  debug
   log_sense!
   minitest
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.2.32
+   2.3.3

data/LICENSE.txt

@@ -1,5 +1,4 @@
-The MIT License (MIT)
-
+The source code is distributed under the terms of the MIT License (MIT)
 Copyright (c) 2021 Shair.Tech
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -19,3 +18,11 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+
+==============================================================================
+LogSense uses data from The free DB-IP Lite database for geolocation purposes.
+
+The DB-IP Lite Database is licensed under a Creative Commons Attribution 4.0
+International License.
+
+For more information: https://db-ip.com 

data/README.org

@@ -14,13 +14,13 @@ LogSense reports the following data:
 - Visitors, hits, unique visitors, bandwidth used
 - Most accessed HTML pages
 - Most accessed resources  
+- Missed resources (also by IP) which helps highlight
+  potential attacks
 - Response statuses
 - Referers
 - OS, browsers, and devices
-- IP Country location, thanks to the DPIP lite country DB
+- IP Country location, thanks to the DP-IP lite country DB
 - Streaks: resources accessed by a given IP over time
-- Potential attacks: access to resources which are not meant to be
-  served by a web server serving static websites
 - Performance of Rails requests
  
 Filters from the command line allow to analyze specific periods and
@@ -33,6 +33,18 @@ And, of course, the compulsory screenshot:
 #+ATTR_HTML: :width 80%
 [[file:./apache-screenshot.png]]
 
+
+* An important word of warning
+
+[[https://owasp.org/www-community/attacks/Log_Injection][Log poisoning]] is a technique whereby attackers send requests with invalidated
+user input to forge log entries or inject malicious content into the logs.
+
+log_sense sanitizes entries of HTML reports, to try and protect from log
+poisoning.  *Log entries and URLs in SQLite3, however, are not sanitized*:
+they are stored and read from the log.  This is not, in general, an issue,
+unless you use the data from SQLite in environments in which URLs can be
+opened or code executed.
+
 * Motivation
 
 LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] (which
@@ -54,6 +66,7 @@ generated files are then made available on a private area on the web.
   gem install log_sense
   #+end_src
 
+
 * Usage
 
   #+begin_src bash :results raw output :wrap example
@@ -65,37 +78,41 @@ generated files are then made available on a private area on the web.
   Usage: log_sense [options] [logfile ...]
           --title=TITLE                Title to use in the report
       -f, --input-format=FORMAT        Input format (either rails or apache)
+      -i, --input-files=file,file,     Input files (can also be passed directly)
       -t, --output-format=FORMAT       Output format: html, org, txt, sqlite. See below for available formats
       -o, --output-file=OUTPUT_FILE    Output file
       -b, --begin=DATE                 Consider entries after or on DATE
       -e, --end=DATE                   Consider entries before or on DATE
-      -l, --limit=N                    Limit to the N most requested resources (defaults to 900)
-      -w, --width=WIDTH                Maximum width of URL and description columns in text reports
+      -l, --limit=N                    Limit to the N most requested resources (defaults to 100)
+      -w, --width=WIDTH                Maximum width of long columns in textual reports
+      -r, --rows=ROWS                  Maximum number of rows for columns with multiple entries in textual reports
       -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
       -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
+          --verbose                    Inform about progress (prints to STDERR)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.4.1
+  This is version 1.5.2
 
   Output formats
-  rails parsing can produce the following outputs:
-    - sqlite
-    - txt
-    - html
   apache parsing can produce the following outputs:
     - sqlite
+    - html
     - txt
+  rails parsing can produce the following outputs:
+    - sqlite
     - html
+    - txt
   #+end_example
 
 Examples:
 
 #+begin_example sh
 log_sense -f apache -i access.log -t txt > access-data.txt
-log_sense -f rails -i production.log -t html -o performance.txt
+log_sense -f rails -i production.log -t html -o performance.html
 #+end_example
 
+
 * Change Log
 
 See the [[file:CHANGELOG.org][CHANGELOG]] file.
@@ -110,8 +127,8 @@ Concerning the outputs:
 - HTML reports use [[https://get.foundation/][Zurb Foundation]], [[https://www.datatables.net/][Data Tables]], and [[https://vega.github.io/vega-lite/][Vega Light]], which
   are all downloaded from a CDN
 - The textual format is compatible with [[https://orgmode.org/][Org Mode]] and can be further
-  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to (including HTML
-  and PDF)
+  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to, including HTML
+  and PDF, with the word of warning in the section above. 
 
 * Author and Contributors
 
@@ -119,12 +136,12 @@ Concerning the outputs:
 
 * Known Bugs
 
-No known bugs; an unknown number of unknown bugs.
-(See the open issues for the known bugs.)
+No known bugs; an unknown number of unknown bugs.  (See the open issues for
+the known bugs.)
 
 * License
 
-Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
+Source code distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
 
 Geolocation is made possible by the DB-IP.com IP to City database,
 released under a CC license.

data/Rakefile

@@ -9,7 +9,21 @@ end
 require_relative './lib/log_sense/ip_locator.rb'
 
 desc "Convert Geolocation DB to sqlite"
-task :dbip_to_sqlite3, [:filename] do |tasks, args|
-  filename = args[:filename]
-  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+task :dbip_to_sqlite3, [:year_month] do |tasks, args|
+  filename = "./ip_locations/dbip-country-lite-#{args[:year_month]}.csv"
+
+  if !File.exist? filename
+    puts "Error. Could not find: #{filename}"
+    puts
+    puts 'I see the following files:'
+    puts Dir.glob("ip_locations/dbip-country-lite*").map { |x| "- #{x}\n" }
+    puts ''
+    puts '1. Download (if necessary) a more recent version from: https://db-ip.com/db/download/ip-to-country-lite'
+    puts '2. Save downloaded file to ip_locations/'
+    puts '3. Relaunch with YYYY-MM'
+
+    exit
+  else
+    LogSense::IpLocator::dbip_to_sqlite filename
+  end
 end

data/exe/log_sense

@@ -11,11 +11,18 @@ require 'log_sense.rb'
 @options     = LogSense::OptionsParser.parse ARGV
 @output_file = @options[:output_file]
 
-if ARGV.map { |x| File.exist?(x) }.include?(false)
-  warn.puts "Error: input file(s) '#{ARGV.reject { |x| File.exist(x) }.join(', ')}' do not exist"
+#
+# Input files can be gotten from an option and from what remains in
+# ARGV
+#
+@input_filenames = @options[:input_filenames] + ARGV
+@non_existing = @input_filenames.reject { |x| File.exist?(x) }
+
+unless @non_existing.empty?
+  $stderr.puts "Error: input file(s) '#{@non_existing.join(', ')}' do not exist"
   exit 1
 end
-@input_files = ARGV.empty? ? [$stdin] : ARGV.map { |x| File.open(x, 'r') }
+@input_files = @input_filenames.empty? ? [$stdin] : @input_filenames.map { |x| File.open(x, 'r') }
 
 #
 # Parse Log and Track Statistics
@@ -31,35 +38,45 @@ when 'rails'
   parser_klass = LogSense::RailsLogParser
   cruncher_klass = LogSense::RailsDataCruncher
 else
-  warn.puts "Error: input format #{@options[:input_format]} not understood."
+  $stderr.puts "Error: input format #{@options[:input_format]} not understood."
   exit 1
 end
 
+$stderr.puts "Parsing input files..." if @options[:verbose]
 @db = parser_klass.parse @input_files
 
 if @options[:output_format] == 'sqlite'
+  $stderr.puts "Saving to SQLite3..." if @options[:verbose]
   ddb = SQLite3::Database.new(@output_file || 'db.sqlite3')
   b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
   b.step(-1) #=> DONE
   b.finish
 else
+  $stderr.puts "Aggregating data..." if @options[:verbose]
   @data = cruncher_klass.crunch @db, @options
+
+  $stderr.puts "Geolocating..." if @options[:verbose]
   @data = LogSense::IpLocator.geolocate @data
 
+  $stderr.puts "Grouping by country..." if @options[:verbose]
+  country_col = @data[:ips][0].size - 1
+  @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+
   @ended_at = Time.now
   @duration = @ended_at - @started_at
 
   @data = @data.merge({
                         command: @command_line,
+                        filenames: ARGV,
                         log_files: @input_files,
                         started_at: @started_at,
                         ended_at: @ended_at,
                         duration: @duration,
                         width: @options[:width]
                       })
-
   #
   # Emit Output
   #
+  $stderr.puts "Emitting..." if @options[:verbose]
   puts LogSense::Emitter.emit @data, @options
 end

data/lib/log_sense/apache_data_cruncher.rb

@@ -17,15 +17,15 @@ module LogSense
       @total_days = 0
       @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
 
-      @source_files   = db.execute "SELECT distinct(source_file) from LogLine"
+      @source_files   = db.execute 'SELECT distinct(source_file) from LogLine'
 
-      @log_size       = db.execute "SELECT count(datetime) from LogLine"
+      @log_size       = db.execute 'SELECT count(datetime) from LogLine'
       @log_size       = @log_size[0][0]
 
       @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
       @selfpolls_size = @selfpolls_size[0][0]
 
-      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @crawlers_size  = db.execute 'SELECT count(datetime) from LogLine where bot == 1'
       @crawlers_size = @crawlers_size[0][0]
 
       @first_day_requested = options[:from_date]
@@ -35,7 +35,7 @@ module LogSense
       @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
 
       @total_days_in_analysis = 0
-      if @first_day_in_analysis and @last_day_in_analysis
+      if @first_day_in_analysis && @last_day_in_analysis
         @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
       end
 
@@ -45,24 +45,24 @@ module LogSense
       filter = [
         (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
         (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
-        (options[:only_crawlers] ? "bot == 1" : nil),
-        (options[:ignore_crawlers] ? "bot == 0" : nil),
+        (options[:only_crawlers] ? 'bot == 1' : nil),
+        (options[:ignore_crawlers] ? 'bot == 0' : nil),
         (options[:no_selfpolls] ? "ip != '::1'" : nil),
-        "true"
+        'true'
       ].compact.join " and "
 
       mega = 1024 * 1024
       giga = mega * 1024
       tera = giga * 1024
-      
+ 
       # in alternative to sum(size)
       human_readable_size = <<-EOS
-      CASE 
+      CASE
       WHEN sum(size) <  1024 THEN sum(size) || ' B' 
       WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
       WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
       WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
-      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
+      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB'
       END AS size
       EOS
 
@@ -101,6 +101,9 @@ module LogSense
       @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
       @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
+      @missed_pages_by_ip = db.execute "SELECT ip, path, status from LogLine where #{bad_statuses} and #{html_page} and #{filter} limit #{options[:limit]}"
+      @missed_resources_by_ip = db.execute "SELECT ip, path, status from LogLine where #{bad_statuses} and #{filter} limit #{options[:limit]}"
+
       @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
 
       @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"
@@ -113,24 +116,26 @@ module LogSense
 
       @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
       @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
+
+      @combined_platforms = db.execute "SELECT browser, platform, ip, count(datetime), #{human_readable_size} from LogLine where #{filter} group by browser, platform, ip order by count(datetime) desc limit #{options[:limit]}"
+
       @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
       
       @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @streaks = db.execute 'SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime'
       data = {}
 
-      self.instance_variables.each do |variable|
-        var_as_symbol = variable.to_s[1..-1].to_sym
-        data[var_as_symbol] = eval(variable.to_s)
+      instance_variables.each do |variable|
+        var_as_symbol = variable.to_s[1..].to_sym
+        data[var_as_symbol] = instance_variable_get(variable)
       end
+
       data
     end
 
-    private
-
-    def self.date_intersect date1, date2, method
-      if date1 and date2
+    def self.date_intersect(date1, date2, method)
+      if date1 && date2
         [date1, date2].send(method)
       elsif date1
         date1
@@ -140,4 +145,3 @@ module LogSense
     end
   end
 end
-

data/lib/log_sense/apache_log_line_parser.rb

@@ -44,7 +44,7 @@ module LogSense
 
     attr_reader :format
 
-    def initialize 
+    def initialize
       @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
     end
 

data/lib/log_sense/apache_log_parser.rb

@@ -82,7 +82,7 @@ module LogSense
               line_number
             )
           rescue StandardError => e
-            warn.puts e.message
+            $stderr.puts e.message
           end
         end
       end