log_sense 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f20a0d2041df1f2414bd0015fff27764c28a89dfd2d45fdb275141638bc5784
-  data.tar.gz: 14426b7383fe9b0077c2852f177545384a369f35489aa5d95372c20cbe19732d
+  metadata.gz: 7d269dedfbb6ec6eae3a77491cc5ec7ca6241f388f2658964541cdd3983b8298
+  data.tar.gz: 6f24d23c8d06430b3605aad90522e08818cd18fd6c71c5fe90823cdc9483c81e
 SHA512:
-  metadata.gz: 9defcff35a5b3802d7d1a7db596a30aa01e28d3b9d784619de5f61713a587e427ccce76fcfcd478e946a0822af8d0822b23bad6392c2a71690ccd07250d5cfb7
-  data.tar.gz: 8b9143feb4f7508de9b7f60eddacafc4713c064f8bda21ed9ae92de364aa5e94fed36ff6d12f2b2aa108337d92c25868bce21d172397f48ff3c41ed93dc9a2b5
+  metadata.gz: a63f715b281101a6f61029da3e2bcf5db4d47a537af562507b0184d6c6755eed436afed729c31cc95255bf064cbe9f487917c96d78199bbee25e6d4189469951
+  data.tar.gz: 77c62a24c3c81067dd0288ad606b732e0f112d0d1710b326be9e894aa72a93db4cb0a188c93b54ada728c2eeb0cf7378fb7033e13982c9a0932414c8455d2703

data/CHANGELOG.org

@@ -2,6 +2,56 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.6.0
+
+- [User] New output format =ufw= generates directives to blacklist IPs
+  requesting URLs matching a pattern. For users of the Uncomplicated
+  Firewall.
+- [User] new option =--no-geo= skips geolocation, which is terribly
+  costly in the current implementation.
+- [User] Updated DB-IP country file to Dec 2022 version.
+- [User] Changed name of SQLite output format to sqlite3
+- [User] It is now possible to start analysis from a sqlite3 DB
+  generated by log_sense, breaking parsing and generation in two
+  steps.
+- [User] Check for correctness of I/O formats before launching
+  analysis
+- [User] Streak report has been renames Session.  Limited the number
+  of URLs shown in each session, to avoid buffer?/memory overflows
+  when an IP requests a massive amount of URLs.
+- [User] Added an IP-per-hour visits report.
+- [Code] A rather extensive refactoring of the source code to 
+  remove code duplications and improve code structure. 
+- [Code] Rubocop-ped various files
+- [Code] Added text renderer to DataTable, which sanitizes input and
+  further reduces risks of XSS and log poisoning attacks
+- [Code] CDN links have been ported into the Emitter module and used
+  in the Embedded Ruby Templates (erbs).  This simplifies version
+  updates of Javascript libraries used in reports.
+
+* 1.5.2
+
+- [User] Updated DB-IP country file.
+- [User] Added reports "Missed Pages by IP" and "Missed Resources by
+  IP".  It can help pinpoint attack sources.
+- [User] Added report "Combined Platform", which puts together
+  Browser, OS, and IP.
+- [User] Summary now shows total size transferred.
+- [User] Added link to DB-IP page for IPs in some tables.
+- [User] Added count of IPs by Country.
+- [User] Improved textual report: values in cells holding multiple
+  values (e.g. IPs) are now shown in distinct lines in the cell. A new
+  option -r limits the number of lines shown per cell.
+- [Default] The number of rows initially shown in HTML reports is now 25.
+- [Default] Default for number of entries in textual report is now
+  100 (used to be 900).
+- [Fixed] The size column in HTML reports is now sorted numerically.
+- [Code] Improved performances of DataTable rendering, using the
+  dataRender flag.
+- [Code] Use trim_mode in ERB to avoid empty lines in HTML output.
+- [Code] Moved to the debug gem.
+- [Gem] Updated email and author's name.
+
 * 1.5.1
 
 - [User] Option --input-files allows to specify input files
@@ -16,7 +66,7 @@
 - [User] Present Unique Visits / day as integer
 - [User] Added Country and Streaks report for rails
 - [User] Changed Streak report in Apache
-- [Gem] Updated DBIP
+- [Gem] Updated DB-IP
 - [Gem] Updated Bundle  
 - [Code] Refactored all reports, so that they are specified
   in the same way  

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.4.2)
+    log_sense (1.5.3)
       browser
       ipaddr
       iso_country_codes
@@ -12,21 +12,30 @@ GEM
   remote: https://rubygems.org/
   specs:
     browser (5.3.1)
-    byebug (11.1.3)
-    ipaddr (1.2.4)
+    debug (1.6.2)
+      irb (>= 1.3.6)
+      reline (>= 0.3.1)
+    io-console (0.5.11)
+    ipaddr (1.2.5)
+    irb (1.4.1)
+      reline (>= 0.3.0)
     iso_country_codes (0.7.8)
+    mini_portile2 (2.8.0)
     minitest (5.15.0)
     rake (12.3.3)
-    sqlite3 (1.4.2)
+    reline (0.3.1)
+      io-console (~> 0.5)
+    sqlite3 (1.5.4)
+      mini_portile2 (~> 2.8.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.1.0)
+    unicode-display_width (2.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  byebug
+  debug
   log_sense!
   minitest
   rake (~> 12.0)

data/LICENSE.txt

@@ -1,5 +1,4 @@
-The MIT License (MIT)
-
+The source code is distributed under the terms of the MIT License (MIT)
 Copyright (c) 2021 Shair.Tech
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -19,3 +18,11 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
+
+==============================================================================
+LogSense uses data from The free DB-IP Lite database for geolocation purposes.
+
+The DB-IP Lite Database is licensed under a Creative Commons Attribution 4.0
+International License.
+
+For more information: https://db-ip.com 

data/README.org

@@ -9,29 +9,62 @@ Rails logs.  Written in Ruby, it runs from the command line, it is
 fast, and it can be installed on any system with a relatively recent
 version of Ruby.  We tested on Ruby 2.6.9, Ruby 3.0.x and later.
 
-LogSense reports the following data:
+When generating reports, LogSense reports the following data:
 
 - Visitors, hits, unique visitors, bandwidth used
 - Most accessed HTML pages
 - Most accessed resources  
+- Missed resources (also by IP) which helps highlight
+  potential attacks
 - Response statuses
 - Referers
 - OS, browsers, and devices
-- IP Country location, thanks to the DPIP lite country DB
+- IP Country location, thanks to the DP-IP lite country DB
 - Streaks: resources accessed by a given IP over time
 - Performance of Rails requests
+
+A special output format =ufw= generates rules for the [[https://launchpad.net/ufw][Uncomplicated
+Firewall]] to blacklist IPs requesting URLs matching a specific pattern.
  
 Filters from the command line allow to analyze specific periods and
 distinguish traffic generated by self polls and crawlers.
 
-LogSense generates HTML, txt, and SQLite outputs.
+LogSense generates HTML, txt, ufw, and SQLite outputs.
 
-And, of course, the compulsory screenshot:
+** Apache Report Structure
 
 #+ATTR_HTML: :width 80%
-[[file:./apache-screenshot.png]]
+[[file:./screenshots/apache-screenshot.png]]
+
+
+** Rails Report Structure
+
+#+ATTR_HTML: :width 80%
+[[file:./screenshots/rails-screenshot.png]]
+
+
+** UFW Report
 
+The output format =ufw= generates directives for Uncomplicated
+Firewall blacklisting IPs requesting URLs matching a given pattern.
 
+We use it to blacklist IPs requesting WordPress login pages on our
+websites... since we don't use WordPress for our websites.
+
+*Example*
+
+#+begin_src 
+$ log_sense -f apache -t ufw -i apache.log
+# /users/sign_in/xmlrpc.php?rsd
+ufw deny from 20.212.3.206
+
+# /wp-login.php /wordpress/wp-login.php /blog/wp-login.php /wp/wp-login.php
+ufw deny from 185.255.134.18
+
+...
+#+end_src
+
+   
 * An important word of warning
 
 [[https://owasp.org/www-community/attacks/Log_Injection][Log poisoning]] is a technique whereby attackers send requests with invalidated
@@ -46,9 +79,10 @@ opened or code executed.
 * Motivation
 
 LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] (which
-strongly inspired the development of Log Sense) and [[https://umami.is/][Umami]], focusing on
-*privacy* and *data-ownership*: the data generated by LogSense is
-stored on your computer and owned by you (like it should be)[fn:1].
+strongly inspired the development of Log Sense) and [[https://umami.is/][Umami]], both
+focusing on *privacy* and *data-ownership*: the data generated by
+LogSense is stored on your computer and owned by you (like it should
+be)[fn:1].
 
 LogSense is also inspired by *static websites generators*: statistics
 are generated from the command line and accessed as static HTML files.
@@ -74,41 +108,84 @@ generated files are then made available on a private area on the web.
   #+RESULTS:
   #+begin_example
   Usage: log_sense [options] [logfile ...]
-          --title=TITLE                Title to use in the report
-      -f, --input-format=FORMAT        Input format (either rails or apache)
-      -i, --input-files=file,file,     Input files (can also be passed directly)
-      -t, --output-format=FORMAT       Output format: html, org, txt, sqlite. See below for available formats
-      -o, --output-file=OUTPUT_FILE    Output file
-      -b, --begin=DATE                 Consider entries after or on DATE
-      -e, --end=DATE                   Consider entries before or on DATE
-      -l, --limit=N                    Limit to the N most requested resources (defaults to 900)
-      -w, --width=WIDTH                Maximum width of URL and description columns in text reports
-      -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
-      -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
-          --verbose                    Inform about progress (prints to STDERR)
-      -v, --version                    Prints version information
-      -h, --help                       Prints this help
-
-  This is version 1.5.1
-
-  Output formats
-  rails parsing can produce the following outputs:
-    - sqlite
-    - txt
-    - html
-  apache parsing can produce the following outputs:
-    - sqlite
-    - txt
-    - html
+        --title=TITLE                Title to use in the report
+    -f, --input-format=FORMAT        Input format (either rails or apache)
+    -i, --input-files=file,file,     Input files (can also be passed directly)
+    -t, --output-format=FORMAT       Output format: html, org, txt, sqlite.
+    -o, --output-file=OUTPUT_FILE    Output file
+    -b, --begin=DATE                 Consider entries after or on DATE
+    -e, --end=DATE                   Consider entries before or on DATE
+    -l, --limit=N                    Limit to the N most requested resources (defaults to 100)
+    -w, --width=WIDTH                Maximum width of long columns in textual reports
+    -r, --rows=ROWS                  Maximum number of rows for columns with multiple entries in textual reports
+    -p, --pattern=PATTERN            Pattern to use with ufw report to decide IP to blacklist
+    -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
+        --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
+    -n, --no-geog                    Do not geolocate entries
+        --verbose                    Inform about progress (output to STDERR)
+    -v, --version                    Prints version information
+    -h, --help                       Prints this help
+
+  This is version 1.6.0
+
+  Output formats:
+
+  - rails: txt, html, sqlite3, ufw
+  - apache: txt, html, sqlite3, ufw
   #+end_example
 
 Examples:
 
 #+begin_example sh
 log_sense -f apache -i access.log -t txt > access-data.txt
-log_sense -f rails -i production.log -t html -o performance.txt
+log_sense -f rails -i production.log -t html -o performance.html
 #+end_example
 
+* Code Structure
+
+The code implements a pipeline, with the following steps:
+
+  1. *Parser:* parses a log to a SQLite3 database. The database
+     contains a table with a list of events, and, in the case of Rails
+     report, a table with the errors.
+  2. *Aggregator:* takes as input a SQLite DB and aggregates data,
+      typically performing "group by", which are simpler to generate in
+      Ruby, rather than in SQL.  The module outputs a Hash, with
+      different reporting data.
+  3. *GeoLocator:* add country information to all the reporting data
+      which has an IP as one the fields.
+  4. *Shaper:* makes (geolocated) aggregated data (e.g. Hashes and
+      such), into Array of Arrays, simplifying the structure of the code
+      building the reports.
+  5. *Emitter* generates reports from shaped data using ERB.
+
+The architecture and the structure of the code is far from being nice,
+for historical reason and for a bunch of small differences existing
+between the input and the outputs to be generated.  This usually ends
+up with modifications to the code that have to be replicated in
+different parts of the code and in interferences.
+
+Among the points I would like to address:
+
+- The execution pipeline in the main script has a few exceptions to
+  manage SQLite reading/dumping and ufw report.  A linear structure
+  would be a lot nicer.
+- Two different classes are defined for steps 1, 2, and 4, to manage,
+  respectively, Apache and Rails logs.  These classes inherit from a
+  common ancestor (e.g. ApacheParser and RailsParser both inherit from
+  Parser), but there is still too little code shared.  A nicer
+  approach would be that of identifying a common DB structure and
+  unify the pipeline up to (or including) the generation of
+  reports. There are a bunch of small different things to highlight in
+  reports, which still make this difficult.  For instance, the country
+  report for Apache reports size of TX data, which is not available
+  for Rail reports.
+- Geolocation could become a lot more efficient if performed in
+  SQLite, rather than in Ruby
+- The distinction between Aggregation, Shaping, and Emission is a too
+  fine-grained and it would be nice to be able to cleanly remove one
+  of the steps.
+
 
 * Change Log
 
@@ -138,7 +215,7 @@ the known bugs.)
 
 * License
 
-Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
+Source code distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
 
 Geolocation is made possible by the DB-IP.com IP to City database,
 released under a CC license.

data/Rakefile

@@ -9,18 +9,18 @@ end
 require_relative './lib/log_sense/ip_locator.rb'
 
 desc "Convert Geolocation DB to sqlite"
-task :dbip_to_sqlite3, [:year_month] do |tasks, args|
-  filename = "./ip_locations/dbip-country-lite-#{args[:year_month]}.csv"
+task :dbip, [:filename] do |tasks, args|
+  filename = args[:filename]
 
   if !File.exist? filename
     puts "Error. Could not find: #{filename}"
     puts
     puts 'I see the following files:'
     puts Dir.glob("ip_locations/dbip-country-lite*").map { |x| "- #{x}\n" }
-    puts ''
-    puts '1. Download (if necessary) a more recent version from: https://db-ip.com/db/download/ip-to-country-lite'
-    puts '2. Save downloaded file to ip_locations/'
-    puts '3. Relaunch with YYYY-MM'
+    puts
+    puts "1. Download (if necessary) a more recent version from: https://db-ip.com/db/download/ip-to-country-lite"
+    puts "2. Save downloaded file to ip_locations/"
+    puts "3. Relaunch with YYYY-MM"
 
     exit
   else

data/exe/log_sense

@@ -1,82 +1,153 @@
 #!/usr/bin/env ruby
 
-require 'log_sense.rb'
+require "log_sense"
+require "sqlite3"
 
 #
 # Parse Command Line Arguments
 #
 
 # this better be here... OptionsParser consumes ARGV
-@command_line = ARGV.join(' ')
-@options     = LogSense::OptionsParser.parse ARGV
-@output_file = @options[:output_file]
+@command_line = ARGV.join(" ")
+@options = LogSense::OptionsParser.parse ARGV
+@input_filenames = @options[:input_filenames] + ARGV
+@output_filename = @options[:output_filename]
 
 #
-# Input files can be gotten from an option and from what remains in
-# ARGV
+# Check correctness of input data.
+#
+
+#
+# Check input files
 #
-@input_filenames = @options[:input_filenames] + ARGV
 @non_existing = @input_filenames.reject { |x| File.exist?(x) }
 
-unless @non_existing.empty?
-  $stderr.puts "Error: input file(s) '#{@non_existing.join(', ')}' do not exist"
+if @non_existing.any?
+  warn "Error: some input file(s) \"#{@non_existing.join(", ")}\" do not exist"
+  exit 1
+end
+
+#
+# Special condition: sqlite3 requires a single file as input
+#
+if @input_filenames.size > 0 &&
+   File.extname(@input_filenames.first) == "sqlite3" &&
+   @input_filenames.size > 1
+  warn "Error: you can pass only one sqlite3 file as input"
+  exit 1
+end
+
+#
+# Supported input/output chains
+#
+iformat = @options[:input_format]
+oformat = @options[:output_format]
+
+if !LogSense::OptionsChecker::compatible?(iformat, oformat)
+  warn "Error: don't know how to make #{iformat} into #{oformat}."
+  warn "Possible transformation chains:"
+  warn LogSense::OptionsChecker.chains_to_s
   exit 1
 end
-@input_files = @input_filenames.empty? ? [$stdin] : @input_filenames.map { |x| File.open(x, 'r') }
 
 #
-# Parse Log and Track Statistics
+# Do the work
 #
 
 @started_at = Time.now
 
-case @options[:input_format]
-when 'apache'
-  parser_klass = LogSense::ApacheLogParser
-  cruncher_klass = LogSense::ApacheDataCruncher
-when 'rails'
-  parser_klass = LogSense::RailsLogParser
-  cruncher_klass = LogSense::RailsDataCruncher
+if @input_filenames.size > 0 &&
+   File.extname(@input_filenames.first) == ".sqlite3"
+  warn "Reading SQLite3 DB ..." if @options[:verbose]
+  @db = SQLite3::Database.open @input_filenames.first
 else
-  $stderr.puts "Error: input format #{@options[:input_format]} not understood."
-  exit 1
+  warn "Parsing ..." if @options[:verbose]
+  @input_files = if @input_filenames.empty?
+                   [$stdin]
+                 else
+                   @input_filenames.map { |fname| File.open(fname, "r") }
+                 end
+  class_name = "LogSense::#{@options[:input_format].capitalize}LogParser"
+  parser_class = Object.const_get class_name
+  parser = parser_class.new
+  @db = parser.parse @input_files
 end
 
-$stderr.puts "Parsing input files..." if @options[:verbose]
-@db = parser_klass.parse @input_files
+if @options[:output_format] == "sqlite3"
+  warn "Saving SQLite3 DB ..." if @options[:verbose]
 
-if @options[:output_format] == 'sqlite'
-  $stderr.puts "Saving to SQLite3..." if @options[:verbose]
-  ddb = SQLite3::Database.new(@output_file || 'db.sqlite3')
-  b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
+  ddb = SQLite3::Database.new(@output_filename || "db.sqlite3")
+  b = SQLite3::Backup.new(ddb, "main", @db, "main")
   b.step(-1) #=> DONE
   b.finish
+
+  exit 0
+elsif @options[:output_format] == "ufw"
+  pattern = @options[:pattern] || "php"
+
+  if @options[:input_format] == "rails"
+    query = "select distinct event.ip,event.url
+                    from error join event
+                    where event.log_id = error.log_id and
+                          event.url like '%#{pattern}%'"
+  else
+    query = "select distinct ip,path from logline
+                    where path like '%#{pattern}%'"
+  end
+
+  ips = @db.execute query
+  ips_and_urls = ips.group_by { |x| x[0] }.transform_values { |x|
+    x.map { |y| y[1..-1] }.flatten
+  }
+  ips_and_urls.each do |ip, urls|
+    puts "# #{urls[0..10].uniq.join(' ')}"
+    puts "ufw deny from #{ip}"
+    puts
+  end
+
+  exit 0
 else
-  $stderr.puts "Aggregating data..." if @options[:verbose]
-  @data = cruncher_klass.crunch @db, @options
+  warn "Aggregating data ..." if @options[:verbose]
+  class_name = "LogSense::#{@options[:input_format].capitalize}Aggregator"
+  aggr_class = Object.const_get class_name
+  aggr = aggr_class.new(@db, @options)
+  @data = aggr.aggregate
 
-  $stderr.puts "Geolocating..." if @options[:verbose]
-  @data = LogSense::IpLocator.geolocate @data
+  if @options[:geolocation]
+    warn "Geolocating ..." if @options[:verbose]
+    @data = LogSense::IpLocator.geolocate @data
 
-  $stderr.puts "Grouping by country..." if @options[:verbose]
-  country_col = @data[:ips][0].size - 1
-  @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+    warn "Grouping IPs by country ..." if @options[:verbose]
+    country_col = @data[:ips][0].size - 1
+    @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+  else
+    @data[:countries] = {}
+  end
 
   @ended_at = Time.now
   @duration = @ended_at - @started_at
 
   @data = @data.merge({
                         command: @command_line,
-                        filenames: ARGV,
+                        filenames: @input_filenames,
                         log_files: @input_files,
                         started_at: @started_at,
                         ended_at: @ended_at,
                         duration: @duration,
                         width: @options[:width]
                       })
-  #
-  # Emit Output
-  #
-  $stderr.puts "Emitting..." if @options[:verbose]
-  puts LogSense::Emitter.emit @data, @options
+
+  if @options[:verbose]
+    warn "I have the following keys in data: "
+    warn @data.keys.sort.map { |key| "#{key}: #{@data[key].class}" }.join("\n")
+  end
+
+  warn "Shaping data for output ..." if @options[:verbose]
+  class_name = "LogSense::#{@options[:input_format].capitalize}ReportShaper"
+  shaper_class = Object.const_get class_name
+  shaper = shaper_class.new
+  @reports = shaper.shape @data
+
+  warn "Emitting..." if @options[:verbose]
+  puts LogSense::Emitter.emit @reports, @data, @options
 end