log_sense 1.6.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a915af429fe2492f17bc767c86767e38d37a674af6206fb3b2c0a76e4cad620
-  data.tar.gz: bf4cc3a1e438b752c9c99fee5f91c85abd38de95dac947c2382e2f9825b51467
+  metadata.gz: 40a7ce48ddf73fc434bd0df0265e5d16d39e5034b62714f0324ea2a8b56950ca
+  data.tar.gz: c6e4ce814d0276f3a2c2aeaa20051772695c8bf8c8f53cbe10440415bb301dec
 SHA512:
-  metadata.gz: 5612ef5474aa397132527588d289d9d1eba4f8954b92553e32fd5b856f2bd5441ba09d89d1bf12a0f220c602dcd2e73de2d6e360b6177b162d57adc2726442c9
-  data.tar.gz: c3b546cc177a3364b1b513f4274c1521aa1669bb17b142eb453ba73251f1e3458e7b9d1703b692ef275a474821ce7375e387155f63e3e7d5d7c9c42e1c50a150
+  metadata.gz: 3b31c02a784d0e3059500f774b090fd40e099d1facd2382195096194ccdecebb5ce601b26b6c04874cc7446e6e625e91be15966d425fc050f3b0be50c67a6985
+  data.tar.gz: 9b79e356ebcaa21b207c24c9ba4c78e39c6865deb4925413aca51b0ce1b705918ca30b61aa205be447547ac428e038239807f82046a9842f7a9d7753037c5fa5

data/CHANGELOG.org

@@ -2,6 +2,23 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.8.0
+
+- Various improvements to the Rails reports
+- Support for BrowserInfo data
+
+* 1.7.0
+
+- [User] Fixes a bug with the geolocator
+- [User] Fixes a bug causing a crash when no country was found by the geolocator
+- [User] Fixes bugs related to corner cases (empty logs, wrong parser for log,
+  empty geolocation data)
+- [User] Updated DB-IP country file to Jun 2024 version.
+- [User] Refreshed the style a bit, removed Fira Sans and updated versions of
+  CSS and JS frameworks
+- [Code] Move options and some code in their own dir
+- [Code] Add rendering view parsing (useful in development; no views yet)
+
 * 1.6.1
 
 - Country DB now stores country name.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.5.3)
+    log_sense (1.7.0)
       browser
       ipaddr
       iso_country_codes
@@ -12,24 +12,30 @@ GEM
   remote: https://rubygems.org/
   specs:
     browser (5.3.1)
-    debug (1.6.2)
-      irb (>= 1.3.6)
-      reline (>= 0.3.1)
-    io-console (0.5.11)
-    ipaddr (1.2.5)
-    irb (1.4.1)
-      reline (>= 0.3.0)
+    debug (1.9.2)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
+    io-console (0.7.2)
+    ipaddr (1.2.6)
+    irb (1.13.1)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
     iso_country_codes (0.7.8)
-    mini_portile2 (2.8.0)
-    minitest (5.15.0)
+    mini_portile2 (2.8.7)
+    minitest (5.23.1)
+    psych (5.1.2)
+      stringio
     rake (12.3.3)
-    reline (0.3.1)
+    rdoc (6.7.0)
+      psych (>= 4.0.0)
+    reline (0.5.8)
       io-console (~> 0.5)
-    sqlite3 (1.5.4)
+    sqlite3 (2.0.2)
       mini_portile2 (~> 2.8.0)
+    stringio (3.1.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.3.0)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   ruby
@@ -41,4 +47,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.3.3
+   2.5.3

data/README.org

@@ -1,11 +1,15 @@
-#+TITLE: README
+#+TITLE: LogSense Readme - Monitor your Rails app easy and fast
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
 * Introduction
 
-LogSense generates reports and statistics from Apache and Ruby on
-Rails logs.  Written in Ruby, it runs from the command line, it is
+LogSense generates reports and statistics from Apache and Ruby on Rails log
+files.  All the statistics you need to monitor your application, its
+performances, and how users access your app.  Since it collects data from logs,
+there is no need for cookies or other tracking technologies.
+
+LogSense is Written in Ruby, it runs from the command line, it is
 fast, and it can be installed on any system with a relatively recent
 version of Ruby.  We tested on Ruby 2.6.9, Ruby 3.0.x and later.
 
@@ -22,6 +26,11 @@ When generating reports, LogSense reports the following data:
 - IP Country location, thanks to the DP-IP lite country DB
 - Streaks: resources accessed by a given IP over time
 - Performance of Rails requests
+- Rails Fatal Errors (with reference to the logs)
+
+LogSense parses also the data generated by BrowserInfo, providing additional
+information for Rails apps, including devices and platforms and number of
+accesses to methods by device type.
 
 A special output format =ufw= generates rules for the [[https://launchpad.net/ufw][Uncomplicated
 Firewall]] to blacklist IPs requesting URLs matching a specific pattern.
@@ -31,16 +40,16 @@ distinguish traffic generated by self polls and crawlers.
 
 LogSense generates HTML, txt, ufw, and SQLite outputs.
 
-** Apache Report Structure
+** Rails Report Structure
 
 #+ATTR_HTML: :width 80%
-[[file:./screenshots/apache-screenshot.png]]
+[[file:./screenshots/rails-screenshot.png]]
 
 
-** Rails Report Structure
+** Apache Report Structure
 
 #+ATTR_HTML: :width 80%
-[[file:./screenshots/rails-screenshot.png]]
+[[file:./screenshots/apache-screenshot.png]]
 
 
 ** UFW Report
@@ -78,19 +87,16 @@ opened or code executed.
 
 * Motivation
 
-LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] (which
-strongly inspired the development of Log Sense) and [[https://umami.is/][Umami]], both
-focusing on *privacy* and *data-ownership*: the data generated by
-LogSense is stored on your computer and owned by you (like it should
-be)[fn:1].
-
-LogSense is also inspired by *static websites generators*: statistics
-are generated from the command line and accessed as static HTML files.
-LogSense thus significantly reduces the attack surface of your
-web server and installation headaches.  We have, for instance, a cron
-job running on our servers, generating statistics at night.  The
-generated files are then made available on a private area on the web.
+LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] and [[https://umami.is/][Umami]], focusing on
+*privacy*, *data-ownership*, and *simplicity*: no need to install JavaScript
+snippets, no tracking cookies, just plain and simple log analysis.
 
+LogSense is also inspired by *static websites generators*: statistics are
+generated from the command line and accessed as static HTML files.  This
+significantly reduces the attack surface of your web server and installation
+headaches.  We have, for instance, a cron job running on our servers, generating
+statistics at night.  The generated files are then made available on a private
+area on the web.
 
 * Installation
 
@@ -98,7 +104,6 @@ generated files are then made available on a private area on the web.
   gem install log_sense
   #+end_src
 
-
 * Usage
 
   #+begin_src bash :results raw output :wrap example
@@ -108,25 +113,25 @@ generated files are then made available on a private area on the web.
   #+RESULTS:
   #+begin_example
   Usage: log_sense [options] [logfile ...]
-        --title=TITLE                Title to use in the report
-    -f, --input-format=FORMAT        Input format (either rails or apache)
-    -i, --input-files=file,file,     Input files (can also be passed directly)
-    -t, --output-format=FORMAT       Output format: html, org, txt, sqlite.
-    -o, --output-file=OUTPUT_FILE    Output file
-    -b, --begin=DATE                 Consider entries after or on DATE
-    -e, --end=DATE                   Consider entries before or on DATE
-    -l, --limit=N                    Limit to the N most requested resources (defaults to 100)
-    -w, --width=WIDTH                Maximum width of long columns in textual reports
-    -r, --rows=ROWS                  Maximum number of rows for columns with multiple entries in textual reports
-    -p, --pattern=PATTERN            Pattern to use with ufw report to decide IP to blacklist
-    -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
-        --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
-    -n, --no-geog                    Do not geolocate entries
-        --verbose                    Inform about progress (output to STDERR)
-    -v, --version                    Prints version information
-    -h, --help                       Prints this help
-
-  This is version 1.6.0
+          --title=TITLE                Title to use in the report
+      -f, --input-format=FORMAT        Log format (stored in log or sqlite3): rails or apache (DEFAULT: apache)
+      -i, --input-files=file,file,     Input file(s), log file or sqlite3 (can also be passed as arguments)
+      -t, --output-format=FORMAT       Output format: html, txt, sqlite, ufw (DEFAULT: html)
+      -o, --output-file=OUTPUT_FILE    Output file. (DEFAULT: STDOUT)
+      -b, --begin=DATE                 Consider only entries after or on DATE
+      -e, --end=DATE                   Consider only entries before or on DATE
+      -l, --limit=N                    Limit to the N most requested resources (DEFAULT: 100)
+      -w, --width=WIDTH                Maximum width of long columns in textual reports
+      -r, --rows=ROWS                  Maximum number of rows for columns with multiple entries in textual reports
+      -p, --pattern=PATTERN            Pattern to use with ufw report to select IP to blacklist (DEFAULT: php)
+      -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
+          --no-selfpoll                Ignore self poll entries (requests from ::1; applies to Apache Logs) (DEFAULT: false)
+          --no-geo                     Do not geolocate entries (DEFAULT: true)
+          --verbose                    Inform about progress (output to STDERR) (DEFAULT: false)
+      -v, --version                    Prints version information
+      -h, --help                       Prints this help
+
+  This is version 1.8.0
 
   Output formats:
 
@@ -210,8 +215,8 @@ Concerning the outputs:
 
 * Known Bugs
 
-No known bugs; an unknown number of unknown bugs.  (See the open issues for
-the known bugs.)
+We have been running LogSense for quite a few years with no particular issues.
+There are no known bugs; there is an unknown number of unknown bugs.
 
 * License
 
@@ -220,8 +225,3 @@ Source code distributed under the terms of the [[http://opensource.org/licenses/
 Geolocation is made possible by the DB-IP.com IP to City database,
 released under a CC license.
 
-[fn:1] There is a small catch: CSS and JavaScript for layout and plots
-are downloaded from a CDN.  Technically, thus, if you generate HTML
-reports and open them, a request is performed and the CDN might keep a
-track (see [[https://en.wikipedia.org/wiki/Content_delivery_network#Security_and_privacy][CDN Security and Privacy on Wikipedia]] for more details).
-Textual reports don't have this issue.

data/Rakefile

@@ -8,9 +8,9 @@ end
 
 require_relative './lib/log_sense/ip_locator.rb'
 
-desc "Convert Geolocation DB to sqlite"
+desc "Convert Geolocation DB to sqlite (arg YYYY_MM or filename)"
 task :dbip, [:filename] do |tasks, args|
-  filename_or_yyyy_mm = args[:filename]
+  filename_or_yyyy_mm = args[:filename] || ""
 
   filename = if /\d{4}-\d{2}/.match(filename_or_yyyy_mm)
                "ip_locations/dbip-country-lite-#{filename_or_yyyy_mm}.csv"
@@ -18,13 +18,15 @@ task :dbip, [:filename] do |tasks, args|
                filename_or_yyyy_mm
              end
 
-  # if the filename has a .gz extension or a gzipped version of the file 
-  # exists, gunzip it
+  # if the filename passed as argument has a .gz extension or a gzipped version
+  # of the file passed as argument exists, gunzip it
   if File.extname(filename) == ".gz" || File.exist?("#{filename}.gz")
     system "gunzip #{filename}.gz"
   end
 
-  if !File.exist? filename
+  if File.exist? filename
+    LogSense::IpLocator::dbip_to_sqlite filename
+  else
     puts <<-EOS
 Error. Could not find: #{filename}
 
@@ -37,13 +39,10 @@ I see the following files:
 3. Relaunch with YYYY-MM (will build: dbip-country-lite-YYYY-MM.csv)
    or with filename.
 
-Remark. If the filename has the extension .gz or if the
-filename does not exist, but a file with the same name and .gz extension
-exists, it is gunzipped first
+Remark. If the filename has the extension .gz or if the filename does not exist,
+but a file with the same name and .gz extension exists, it is gunzipped first
     EOS
 
     exit
-  else
-    LogSense::IpLocator::dbip_to_sqlite filename
   end
 end

data/exe/log_sense

@@ -9,7 +9,7 @@ require "sqlite3"
 
 # this better be here... OptionsParser consumes ARGV
 @command_line = ARGV.join(" ")
-@options = LogSense::OptionsParser.parse ARGV
+@options = LogSense::Options::Parser.parse ARGV
 @input_filenames = @options[:input_filenames] + ARGV
 @output_filename = @options[:output_filename]
 
@@ -20,33 +20,36 @@ require "sqlite3"
 #
 # Check input files
 #
-@non_existing = @input_filenames.reject { |x| File.exist?(x) }
 
+@non_existing = @input_filenames.reject { |x| File.exist?(x) }
 if @non_existing.any?
   warn "Error: some input file(s) \"#{@non_existing.join(", ")}\" do not exist"
   exit 1
 end
 
-#
-# Special condition: sqlite3 requires a single file as input
-#
-if @input_filenames.size > 0 &&
-   File.extname(@input_filenames.first) == "sqlite3" &&
-   @input_filenames.size > 1
-  warn "Error: you can pass only one sqlite3 file as input"
+@sqlite3_files = @input_filenames.select { |x| File.extname(x).include?("sqlite") }
+if @sqlite3_files.any? && @input_filenames.size != 1
+  warn "Error: when passing an SQLite3 DB, this has to be the only input file"
   exit 1
 end
 
+#
+# Check output files
+#
+
+# Nothing to be done, here, since we output to STDOUT if no output filename is
+# specified
+
 #
 # Supported input/output chains
 #
 iformat = @options[:input_format]
 oformat = @options[:output_format]
 
-if !LogSense::OptionsChecker::compatible?(iformat, oformat)
+if !LogSense::Options::Checker.compatible?(iformat, oformat)
   warn "Error: don't know how to make #{iformat} into #{oformat}."
   warn "Possible transformation chains:"
-  warn LogSense::OptionsChecker.chains_to_s
+  warn LogSense::Options::Checker.chains_to_s
   exit 1
 end
 
@@ -56,8 +59,11 @@ end
 
 @started_at = Time.now
 
-if @input_filenames.size > 0 &&
-   File.extname(@input_filenames.first) == ".sqlite3"
+#
+# Input
+#
+
+if @input_filenames.size > 0 && File.extname(@input_filenames.first) == ".sqlite3"
   warn "Reading SQLite3 DB ..." if @options[:verbose]
   @db = SQLite3::Database.open @input_filenames.first
 else
@@ -67,12 +73,19 @@ else
                  else
                    @input_filenames.map { |fname| File.open(fname, "r") }
                  end
-  class_name = "LogSense::#{@options[:input_format].capitalize}LogParser"
+
+  class_name = "LogSense::#{@options[:input_format].capitalize}::LogParser"
   parser_class = Object.const_get class_name
   parser = parser_class.new
   @db = parser.parse @input_files
 end
 
+#
+# Output
+# 
+
+# TODO this code could benefit from some classes abstracting the work a bit
+
 if @options[:output_format] == "sqlite3"
   warn "Saving SQLite3 DB ..." if @options[:verbose]
 
@@ -83,7 +96,7 @@ if @options[:output_format] == "sqlite3"
 
   exit 0
 elsif @options[:output_format] == "ufw"
-  pattern = @options[:pattern] || "php"
+  pattern = @options[:pattern]
 
   if @options[:input_format] == "rails"
     query = "select distinct event.ip,event.url
@@ -113,14 +126,18 @@ else
   aggr = aggr_class.new(@db, @options)
   @data = aggr.aggregate
 
-  if @options[:geolocation]
+  if @options[:geolocation] && @data[:ips].size != 0
     warn "Geolocating ..." if @options[:verbose]
-    @data = LogSense::IpLocator.geolocate @data
+    geolocated_data = LogSense::IpLocator.geolocate @data
 
     warn "Grouping IPs by country ..." if @options[:verbose]
-    country_col = @data[:ips][0].size - 1
-    @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+    country_col = geolocated_data[0].size - 1
+    @data[:countries] = geolocated_data.group_by { |x| x[country_col] }
+  elsif @options[:geolocation] && @data[:ips].size == 0
+    warn "Skipping geolocation: no IP found" if @options[:verbose]
+    @data[:countries] = {}
   else
+    warn "Skipping geolocation." if @options[:verbose]
     @data[:countries] = {}
   end
 

data/lib/log_sense/apache/log_line_parser.rb

@@ -0,0 +1,59 @@
+module LogSense
+  module Apache
+    # parses a log line and returns a hash
+    # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
+    #
+    # %h: IP
+    # %l: ident or -
+    # %u: userid or -
+    # %t: [10/Oct/2000:13:55:36 -0700]
+    #   day = 2*digit
+    #   month = 3*letter
+    #   year = 4*digit
+    #   hour = 2*digit
+    #   minute = 2*digit
+    #   second = 2*digit
+    #   zone = (`+' | `-') 4*digit
+    # %r: GET /apache_pb.gif HTTP/1.0
+    # %{User-agent}: "
+    #
+    # Example
+    # 116.179.32.16 - - [19/Dec/2021:22:35:11 +0100] "GET / HTTP/1.1" 200 135 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
+    #
+    class LogLineParser
+      DAY = /[0-9]{2}/
+      MONTH = /[A-Za-z]{3}/
+      YEAR = /[0-9]{4}/
+      TIMEC = /[0-9]{2}/
+      TIMEZONE = /(\+|-)[0-9]{4}/
+
+      IP = /(?<ip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|::1|unknown)/
+      IDENT = /(?<ident>[^ ]+|-)/
+      USERID = /(?<userid>[^ ]+|-)/
+
+      TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
+
+      HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
+      WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
+      OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
+      METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
+      PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
+      URL = /(?<url>[^ ]+)/
+      REFERER = /(?<referer>[^"]*)/
+      RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
+      SIZE = /(?<size>[0-9]+|-)/
+      USER_AGENT = /(?<user_agent>[^"]*)/
+
+      attr_reader :format
+
+      def initialize
+        @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
+      end
+
+      def parse(line)
+        @format.match(line) ||
+          raise("Apache LogLine Parser Error: Could not parse #{line}")
+      end
+    end
+  end
+end

data/lib/log_sense/apache/log_parser.rb

@@ -0,0 +1,101 @@
+require "sqlite3"
+require "browser"
+require_relative "log_line_parser"
+
+module LogSense
+  module Apache
+    #
+    # parse an Apache log file and return a SQLite3 DB
+    #
+    class LogParser
+      def parse(streams, options = {})
+        db = SQLite3::Database.new ":memory:"
+
+        db.execute "CREATE TABLE IF NOT EXISTS LogLine(
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      datetime TEXT,
+      ip TEXT,
+      user TEXT,
+      unique_visitor TEXT,
+      method TEXT,
+      path TEXT,
+      extension TEXT,
+      status TEXT,
+      size INTEGER,
+      referer TEXT,
+      user_agent TEXT,
+      bot INTEGER,
+      browser TEXT,
+      browser_version TEXT,
+      platform TEXT,
+      platform_version TEXT,
+      source_file TEXT,
+      line_number INTEGER
+      )"
+
+        ins = db.prepare("insert into LogLine (
+                datetime,
+                ip,
+                user,
+                unique_visitor,
+                method,
+                path,
+                extension,
+                status,
+                size,
+                referer,
+                user_agent,
+                bot,
+                browser,
+                browser_version,
+                platform,
+                platform_version,
+                source_file,
+                line_number
+                )
+              values (#{Array.new(18, '?').join(', ')})")
+
+        parser = LogLineParser.new
+
+        streams.each do |stream|
+          stream.readlines.each_with_index do |line, line_number|
+            begin
+              hash = parser.parse line
+              ua = Browser.new(hash[:user_agent], accept_language: 'en-us')
+              ins.execute(
+                DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
+                hash[:ip],
+                hash[:userid],
+                unique_visitor_id(hash),
+                hash[:method],
+                hash[:url],
+                (hash[:url] ? File.extname(hash[:url]) : ''),
+                hash[:status],
+                hash[:size].to_i,
+                hash[:referer],
+                hash[:user_agent],
+                ua.bot? ? 1 : 0,
+                (ua.name || ''),
+                (ua.version || ''),
+                (ua.platform.name || ''),
+                (ua.platform.version || ''),
+                stream == $stdin ? "stdin" : stream.path,
+                line_number
+              )
+            rescue StandardError => e
+              warn e.message
+            end
+          end
+        end
+
+        db
+      end
+
+      private
+
+      def unique_visitor_id hash
+        "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
+      end
+    end
+  end
+end