log_sense 1.4.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 266e20972553f6d409814398dab832334a4c01bfa12e90c74acfdc75ee0b7c8d
-  data.tar.gz: 228e6bdc2d931e5190d82fc5ba66660ff6f0de0277a876746154de81a1ffe4e2
+  metadata.gz: 1f20a0d2041df1f2414bd0015fff27764c28a89dfd2d45fdb275141638bc5784
+  data.tar.gz: 14426b7383fe9b0077c2852f177545384a369f35489aa5d95372c20cbe19732d
 SHA512:
-  metadata.gz: f1454d78cfec258ff3bc69359be29178ebab4cf7ffd2869d736c29f2cffd6efe209f65be59298864bf94de30bd022a3397c91446b043c49e704b6d38ced59357
-  data.tar.gz: aa7239af4bb17270a23d9931194859b01a8ff50ebb9cc3c3ed37aeac1702aef413051b0381c57208dcffedd31ccbff5bd0a9485c0fe18947f540cda9f4463acd
+  metadata.gz: 9defcff35a5b3802d7d1a7db596a30aa01e28d3b9d784619de5f61713a587e427ccce76fcfcd478e946a0822af8d0822b23bad6392c2a71690ccd07250d5cfb7
+  data.tar.gz: 8b9143feb4f7508de9b7f60eddacafc4713c064f8bda21ed9ae92de364aa5e94fed36ff6d12f2b2aa108337d92c25868bce21d172397f48ff3c41ed93dc9a2b5

data/CHANGELOG.org

@@ -2,6 +2,40 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.5.1
+
+- [User] Option --input-files allows to specify input files
+  in addition to passing filenames to the command line
+- [User] Minor changes to the layout of HTML reports
+- [User] Add version number in reports
+- [Fixed] Duplicated entries in navigation
+- [Code] Updated and added minitest(s)
+
+* 1.5.0
+
+- [User] Present Unique Visits / day as integer
+- [User] Added Country and Streaks report for rails
+- [User] Changed Streak report in Apache
+- [Gem] Updated DBIP
+- [Gem] Updated Bundle  
+- [Code] Refactored all reports, so that they are specified
+  in the same way  
+- [Code] Refactor warning message in textual reports
+- [Code] Build HTML menu for report specification
+- [Code] Various refactoring passes on the code
+
+* 1.4.1
+
+- [User] New textual report for Apache
+- [User] New option -w sets maximum width of URL, Path, and
+  Description columns in textual reports
+- [User] Removed option -i, since input filenames are now taken
+  as direct arguments
+- [User] Allow multiple files in input
+- [Fixed] Complain if input format is not supported
+- [Code] Refactoring of reports to manage better output to
+  multiple formats  
+
 * 1.4.0
 
 - [User] The Apache Log report now organizes page requests in four

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.3.1)
+    log_sense (1.4.2)
       browser
       ipaddr
       iso_country_codes
@@ -13,9 +13,9 @@ GEM
   specs:
     browser (5.3.1)
     byebug (11.1.3)
-    ipaddr (1.2.3)
+    ipaddr (1.2.4)
     iso_country_codes (0.7.8)
-    minitest (5.14.4)
+    minitest (5.15.0)
     rake (12.3.3)
     sqlite3 (1.4.2)
     terminal-table (3.0.2)
@@ -32,4 +32,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.2.32
+   2.3.3

data/README.org

@@ -19,8 +19,6 @@ LogSense reports the following data:
 - OS, browsers, and devices
 - IP Country location, thanks to the DPIP lite country DB
 - Streaks: resources accessed by a given IP over time
-- Potential attacks: access to resources which are not meant to be
-  served by a web server serving static websites
 - Performance of Rails requests
  
 Filters from the command line allow to analyze specific periods and
@@ -33,6 +31,18 @@ And, of course, the compulsory screenshot:
 #+ATTR_HTML: :width 80%
 [[file:./apache-screenshot.png]]
 
+
+* An important word of warning
+
+[[https://owasp.org/www-community/attacks/Log_Injection][Log poisoning]] is a technique whereby attackers send requests with invalidated
+user input to forge log entries or inject malicious content into the logs.
+
+log_sense sanitizes entries of HTML reports, to try and protect from log
+poisoning.  *Log entries and URLs in SQLite3, however, are not sanitized*:
+they are stored and read from the log.  This is not, in general, an issue,
+unless you use the data from SQLite in environments in which URLs can be
+opened or code executed.
+
 * Motivation
 
 LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] (which
@@ -54,6 +64,7 @@ generated files are then made available on a private area on the web.
   gem install log_sense
   #+end_src
 
+
 * Usage
 
   #+begin_src bash :results raw output :wrap example
@@ -62,21 +73,23 @@ generated files are then made available on a private area on the web.
 
   #+RESULTS:
   #+begin_example
-  Usage: log_sense [options] [logfile]
+  Usage: log_sense [options] [logfile ...]
           --title=TITLE                Title to use in the report
       -f, --input-format=FORMAT        Input format (either rails or apache)
-      -i, --input-file=INPUT_FILE      Input file
+      -i, --input-files=file,file,     Input files (can also be passed directly)
       -t, --output-format=FORMAT       Output format: html, org, txt, sqlite. See below for available formats
       -o, --output-file=OUTPUT_FILE    Output file
       -b, --begin=DATE                 Consider entries after or on DATE
       -e, --end=DATE                   Consider entries before or on DATE
-      -l, --limit=N                    Number of entries to show (defaults to 30)
+      -l, --limit=N                    Limit to the N most requested resources (defaults to 900)
+      -w, --width=WIDTH                Maximum width of URL and description columns in text reports
       -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
       -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
+          --verbose                    Inform about progress (prints to STDERR)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.3.1
+  This is version 1.5.1
 
   Output formats
   rails parsing can produce the following outputs:
@@ -85,6 +98,7 @@ generated files are then made available on a private area on the web.
     - html
   apache parsing can produce the following outputs:
     - sqlite
+    - txt
     - html
   #+end_example
 
@@ -95,6 +109,7 @@ log_sense -f apache -i access.log -t txt > access-data.txt
 log_sense -f rails -i production.log -t html -o performance.txt
 #+end_example
 
+
 * Change Log
 
 See the [[file:CHANGELOG.org][CHANGELOG]] file.
@@ -109,8 +124,8 @@ Concerning the outputs:
 - HTML reports use [[https://get.foundation/][Zurb Foundation]], [[https://www.datatables.net/][Data Tables]], and [[https://vega.github.io/vega-lite/][Vega Light]], which
   are all downloaded from a CDN
 - The textual format is compatible with [[https://orgmode.org/][Org Mode]] and can be further
-  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to (including HTML
-  and PDF)
+  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to, including HTML
+  and PDF, with the word of warning in the section above. 
 
 * Author and Contributors
 
@@ -118,8 +133,8 @@ Concerning the outputs:
 
 * Known Bugs
 
-No known bugs; an unknown number of unknown bugs.
-(See the open issues for the known bugs.)
+No known bugs; an unknown number of unknown bugs.  (See the open issues for
+the known bugs.)
 
 * License
 

data/Rakefile

@@ -9,7 +9,21 @@ end
 require_relative './lib/log_sense/ip_locator.rb'
 
 desc "Convert Geolocation DB to sqlite"
-task :dbip_to_sqlite3, [:filename] do |tasks, args|
-  filename = args[:filename]
-  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+task :dbip_to_sqlite3, [:year_month] do |tasks, args|
+  filename = "./ip_locations/dbip-country-lite-#{args[:year_month]}.csv"
+
+  if !File.exist? filename
+    puts "Error. Could not find: #{filename}"
+    puts
+    puts 'I see the following files:'
+    puts Dir.glob("ip_locations/dbip-country-lite*").map { |x| "- #{x}\n" }
+    puts ''
+    puts '1. Download (if necessary) a more recent version from: https://db-ip.com/db/download/ip-to-country-lite'
+    puts '2. Save downloaded file to ip_locations/'
+    puts '3. Relaunch with YYYY-MM'
+
+    exit
+  else
+    LogSense::IpLocator::dbip_to_sqlite filename
+  end
 end

data/exe/log_sense

@@ -7,21 +7,22 @@ require 'log_sense.rb'
 #
 
 # this better be here... OptionsParser consumes ARGV
-@command_line = ARGV.join(" ")
-
+@command_line = ARGV.join(' ')
 @options     = LogSense::OptionsParser.parse ARGV
-@input_file  = @options[:input_file] || ARGV[0]
 @output_file = @options[:output_file]
 
-if not @input_file
-  puts "Error: no input file specified."
-  exit
-end
+#
+# Input files can be gotten from an option and from what remains in
+# ARGV
+#
+@input_filenames = @options[:input_filenames] + ARGV
+@non_existing = @input_filenames.reject { |x| File.exist?(x) }
 
-if not File.exist? @input_file
-  puts "Error: input file '#{@input_file}' does not exist"
+unless @non_existing.empty?
+  $stderr.puts "Error: input file(s) '#{@non_existing.join(', ')}' do not exist"
   exit 1
 end
+@input_files = @input_filenames.empty? ? [$stdin] : @input_filenames.map { |x| File.open(x, 'r') }
 
 #
 # Parse Log and Track Statistics
@@ -36,32 +37,46 @@ when 'apache'
 when 'rails'
   parser_klass = LogSense::RailsLogParser
   cruncher_klass = LogSense::RailsDataCruncher
+else
+  $stderr.puts "Error: input format #{@options[:input_format]} not understood."
+  exit 1
 end
 
-@db = parser_klass.parse @input_file
+$stderr.puts "Parsing input files..." if @options[:verbose]
+@db = parser_klass.parse @input_files
 
-if @options[:output_format]  == "sqlite"
-  ddb = SQLite3::Database.new(@output_file || "db.sqlite3")
+if @options[:output_format] == 'sqlite'
+  $stderr.puts "Saving to SQLite3..." if @options[:verbose]
+  ddb = SQLite3::Database.new(@output_file || 'db.sqlite3')
   b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
   b.step(-1) #=> DONE
   b.finish
 else
+  $stderr.puts "Aggregating data..." if @options[:verbose]
   @data = cruncher_klass.crunch @db, @options
+
+  $stderr.puts "Geolocating..." if @options[:verbose]
   @data = LogSense::IpLocator.geolocate @data
 
+  $stderr.puts "Grouping by country..." if @options[:verbose]
+  country_col = @data[:ips][0].size - 1
+  @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+
   @ended_at = Time.now
   @duration = @ended_at - @started_at
 
   @data = @data.merge({
                         command: @command_line,
-                        log_file: @input_file,
+                        filenames: ARGV,
+                        log_files: @input_files,
                         started_at: @started_at,
                         ended_at: @ended_at,
-                        duration: @duration
+                        duration: @duration,
+                        width: @options[:width]
                       })
-
   #
   # Emit Output
   #
+  $stderr.puts "Emitting..." if @options[:verbose]
   puts LogSense::Emitter.emit @data, @options
 end

data/lib/log_sense/apache_data_cruncher.rb

@@ -15,17 +15,17 @@ module LogSense
       @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
 
       @total_days = 0
-      if @first_day and @last_day
-        @total_days = (@last_day - @first_day).to_i
-      end
+      @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
+
+      @source_files   = db.execute 'SELECT distinct(source_file) from LogLine'
 
-      @log_size       = db.execute "SELECT count(datetime) from LogLine"
+      @log_size       = db.execute 'SELECT count(datetime) from LogLine'
       @log_size       = @log_size[0][0]
 
       @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
       @selfpolls_size = @selfpolls_size[0][0]
 
-      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @crawlers_size  = db.execute 'SELECT count(datetime) from LogLine where bot == 1'
       @crawlers_size = @crawlers_size[0][0]
 
       @first_day_requested = options[:from_date]
@@ -35,7 +35,7 @@ module LogSense
       @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
 
       @total_days_in_analysis = 0
-      if @first_day_in_analysis and @last_day_in_analysis
+      if @first_day_in_analysis && @last_day_in_analysis
         @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
       end
 
@@ -45,24 +45,24 @@ module LogSense
       filter = [
         (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
         (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
-        (options[:only_crawlers] ? "bot == 1" : nil),
-        (options[:ignore_crawlers] ? "bot == 0" : nil),
+        (options[:only_crawlers] ? 'bot == 1' : nil),
+        (options[:ignore_crawlers] ? 'bot == 0' : nil),
         (options[:no_selfpolls] ? "ip != '::1'" : nil),
-        "true"
+        'true'
       ].compact.join " and "
 
       mega = 1024 * 1024
       giga = mega * 1024
       tera = giga * 1024
-      
+ 
       # in alternative to sum(size)
       human_readable_size = <<-EOS
-      CASE 
+      CASE
       WHEN sum(size) <  1024 THEN sum(size) || ' B' 
       WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
       WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
       WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
-      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
+      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB'
       END AS size
       EOS
 
@@ -117,20 +117,19 @@ module LogSense
       
       @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @streaks = db.execute 'SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime'
       data = {}
 
-      self.instance_variables.each do |variable|
-        var_as_symbol = variable.to_s[1..-1].to_sym
-        data[var_as_symbol] = eval(variable.to_s)
+      instance_variables.each do |variable|
+        var_as_symbol = variable.to_s[1..].to_sym
+        data[var_as_symbol] = instance_variable_get(variable)
       end
+
       data
     end
 
-    private
-
-    def self.date_intersect date1, date2, method
-      if date1 and date2
+    def self.date_intersect(date1, date2, method)
+      if date1 && date2
         [date1, date2].send(method)
       elsif date1
         date1
@@ -140,4 +139,3 @@ module LogSense
     end
   end
 end
-

data/lib/log_sense/apache_log_line_parser.rb

@@ -31,20 +31,20 @@ module LogSense
 
     TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
 
-    HTTP_METHODS=/GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
-    WEBDAV_METHODS=/COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-    OTHER_METHODS=/SEARCH|REPORT|PRI|HEAD\/robots.txt/
-    METHOD=/(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-    PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
-    URL=/(?<url>[^ ]+)/
-    REFERER=/(?<referer>[^"]*)/
-    RETURN_CODE=/(?<status>[1-5][0-9][0-9])/
-    SIZE=/(?<size>[0-9]+|-)/
+    HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
+    WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
+    OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
+    METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
+    PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
+    URL = /(?<url>[^ ]+)/
+    REFERER = /(?<referer>[^"]*)/
+    RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
+    SIZE = /(?<size>[0-9]+|-)/
     USER_AGENT = /(?<user_agent>[^"]*)/
 
     attr_reader :format
 
-    def initialize 
+    def initialize
       @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
     end
 

data/lib/log_sense/apache_log_parser.rb

@@ -7,10 +7,9 @@ module LogSense
     # parse an Apache log file and return a SQLite3 DB
     #
 
-    def self.parse filename, options = {}
-      content = filename ? File.readlines(filename) : ARGF.readlines
+    def self.parse(streams, options = {})
+      db = SQLite3::Database.new ':memory:'
 
-      db = SQLite3::Database.new ":memory:"
       db.execute "CREATE TABLE IF NOT EXISTS LogLine(
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       datetime TEXT,
@@ -28,15 +27,18 @@ module LogSense
       browser TEXT,
       browser_version TEXT,
       platform TEXT,
-      platform_version TEXT)"
+      platform_version TEXT,
+      source_file TEXT,
+      line_number INTEGER
+      )"
       
-      ins = db.prepare('insert into LogLine (
-                datetime, 
+      ins = db.prepare("insert into LogLine (
+                datetime,
                 ip,
                 user,
                 unique_visitor,
                 method,
-                path, 
+                path,
                 extension,
                 status,
                 size,
@@ -46,44 +48,50 @@ module LogSense
                 browser,
                 browser_version,
                 platform,
-                platform_version)
-              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+                platform_version,
+                source_file,
+                line_number
+                )
+              values (#{Array.new(18, '?').join(', ')})")
 
       parser = ApacheLogLineParser.new
-      
-      content.each do |line|
-        begin
-          hash = parser.parse line
-          ua = Browser.new(hash[:user_agent], accept_language: "en-us")
-          ins.execute(
-            DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
-            hash[:ip],
-            hash[:userid],
-            unique_visitor_id(hash),
-            hash[:method],
-            hash[:url],
-            (hash[:url] ? File.extname(hash[:url]) : ""),
-            hash[:status],
-            hash[:size].to_i,
-            hash[:referer],
-            hash[:user_agent],
-            ua.bot? ? 1 : 0,
-            (ua.name || ""),
-            (ua.version || ""),
-            (ua.platform.name || ""),
-            (ua.platform.version || "")
-          )
-        rescue StandardError => e
-          STDERR.puts e.message
+
+      streams.each do |stream|
+        stream.readlines.each_with_index do |line, line_number|
+          begin
+            hash = parser.parse line
+            ua = Browser.new(hash[:user_agent], accept_language: 'en-us')
+            ins.execute(
+              DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
+              hash[:ip],
+              hash[:userid],
+              unique_visitor_id(hash),
+              hash[:method],
+              hash[:url],
+              (hash[:url] ? File.extname(hash[:url]) : ''),
+              hash[:status],
+              hash[:size].to_i,
+              hash[:referer],
+              hash[:user_agent],
+              ua.bot? ? 1 : 0,
+              (ua.name || ''),
+              (ua.version || ''),
+              (ua.platform.name || ''),
+              (ua.platform.version || ''),
+              stream == $stdin ? "stdin" : stream.path,
+              line_number
+            )
+          rescue StandardError => e
+            $stderr.puts e.message
+          end
         end
       end
-      
+
       db
     end
 
     def self.unique_visitor_id hash
       "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
     end
-
   end
 end