log_sense 1.9.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ccf396e27466411bfa603709787126290e24d0652ac5cc6b96bad008d11ccd6d
-  data.tar.gz: 4cee6242192c1c38273c7a9fa55c2d55ea18aa65c6328932a27f944fa8b25cbd
+  metadata.gz: e17413654dc552a716548d7a55e127c5f0703cf29643d97d500e0cf51b20eba3
+  data.tar.gz: 56765b54fdda4f64add3fbb97ea3b4073976d25f4cb67cc932a7d350d85793cb
 SHA512:
-  metadata.gz: 25874781f2e012a40de832d553a1a3bd0cfa0fd232988e482a2e6ceaf6c3cdd632bbe5059bbb134f4fc3029add42ff0e51311d9c6c3a61feedebf69df1f9c8b6
-  data.tar.gz: 0e9733a78e5b972ed20c657983e04f7d238a14646e91a0c68e2d4594c4fbaecacc5271ec1c3e15949d590b86f04049b837e448b1473f9db60a8c2a3673074a02
+  metadata.gz: dc77b1c63c08a9a190aae0ab0ab44bdffe37e0dc2e999d3dc8a236703dc8e430fb32ff09d884d5d9e90ff7870b7bb212d0cc486e877982ca3c5328a3aeda5943
+  data.tar.gz: e73ad267ad54dc4e3c09481ec4f055e6d9e1e9346262b04975b754bd8822b19ce27298299ce6fb7cc3511e5248b6303392c984b48d68d40bd398bd14d7be9d80

data/CHANGELOG.org

@@ -2,6 +2,27 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.1.0
+
+- [User] Delayed Job Errors
+- [User] Improve scatter plot about performances (Use labels, swap x and y axes,
+  better tooltip)
+- [User] Updated DB-IP country file to Aug 2024 version (19/08).
+- [User] Fixes CHANGELOG
+- [User] Revises "Fatal Errors" removing a bit of redundancy and collecting all
+  message
+- [Bug/User] Added filtering on date on reports which still did not use it
+
+* 2.0.0
+
+- [User] World Map
+- [User] Dark mode
+- [User] Bars in the statuses bar plot are now colored according to status
+- [User] Add "statuses by day" in Rails report
+- [User] Enlarge "errors" and "potential attacks" reports
+- [Bug] Fix link colors in sidebar
+- [Code] Various smaller fixes
+
 * 1.9.0
 
 - Perform calculation on HTML pages only

data/Gemfile

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in apache_log_report.gemspec
+# Specify your gem's dependencies in log_sense.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    log_sense (1.7.0)
-      browser
-      ipaddr
-      iso_country_codes
-      sqlite3
-      terminal-table
+    log_sense (2.0.0)
+      browser (~> 5.3.0)
+      ipaddr (~> 1.2.0)
+      iso_country_codes (~> 0.7.0)
+      sqlite3 (~> 2.0.0)
+      terminal-table (~> 3.0.0)
 
 GEM
   remote: https://rubygems.org/
@@ -17,22 +17,22 @@ GEM
       reline (>= 0.3.8)
     io-console (0.7.2)
     ipaddr (1.2.6)
-    irb (1.13.1)
+    irb (1.14.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     iso_country_codes (0.7.8)
     mini_portile2 (2.8.7)
-    minitest (5.23.1)
+    minitest (5.24.1)
     psych (5.1.2)
       stringio
-    rake (12.3.3)
+    rake (13.2.1)
     rdoc (6.7.0)
       psych (>= 4.0.0)
-    reline (0.5.8)
+    reline (0.5.9)
       io-console (~> 0.5)
-    sqlite3 (2.0.2)
+    sqlite3 (2.0.3)
       mini_portile2 (~> 2.8.0)
-    stringio (3.1.0)
+    stringio (3.1.1)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     unicode-display_width (2.5.0)
@@ -41,10 +41,10 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  debug
+  debug (~> 1.9.0)
   log_sense!
-  minitest
-  rake (~> 12.0)
+  minitest (~> 5.24.0)
+  rake (~> 13.0)
 
 BUNDLED WITH
    2.5.3

data/README.org

@@ -4,58 +4,90 @@
 
 * Introduction
 
-LogSense generates reports and statistics from Apache and Ruby on Rails log
-files.  All the statistics you need to monitor your application, its
-performances, and how users access your app.  Since it collects data from logs,
-there is no need for cookies or other tracking technologies.
-
-LogSense is Written in Ruby, it runs from the command line, it is
-fast, and it can be installed on any system with a relatively recent
-version of Ruby.  We tested on Ruby 2.6.9, Ruby 3.0.x and later.
-
-When generating reports, LogSense reports the following data:
-
-- Visitors, hits, unique visitors, bandwidth used
-- Most accessed HTML pages
-- Most accessed resources  
-- Missed resources (also by IP) which helps highlight
-  potential attacks
-- Response statuses
-- Referers
-- OS, browsers, and devices
-- IP Country location, thanks to the DP-IP lite country DB
-- Streaks: resources accessed by a given IP over time
-- Performance of Rails requests
-- Rails Fatal Errors (with reference to the logs)
+LogSense generates reports and statistics from Ruby on Rails and Apache/Nginx
+log files.
+
+Main features:
+
+- Statistics for Rails app in production and Web server logs (combined format,
+  which can be produced both by Apache and Nginx)
+- Reports on performances, errors, visitors, and devices used to access your
+  websites and webapps[fn:: LogSense parses also the data generated by the
+  BrowserInfo gem, providing additional information for Rails apps, including
+  devices, platforms and number of accesses to methods by device type.].
+- Can combine one or more log files
+- No need for cookies or other tracking technologies (but you need access to
+  your log files)
+- Filters allow to analyze specific periods distinguish traffic generated by
+  self polls and crawlers.
+- Reports can be generated in HTML, txt, ufw, and SQLite.  HTML reports are
+  responsive and come with dark and light theme.
 
-LogSense parses also the data generated by BrowserInfo, providing additional
-information for Rails apps, including devices and platforms and number of
-accesses to methods by device type.
+LogSense is Written in Ruby, it runs from the command line, it is fast, and it
+can be installed on any system with a relatively recent version of Ruby.  We
+use it with Ruby 3.1.4 and 3.3.0.
 
-A special output format =ufw= generates rules for the [[https://launchpad.net/ufw][Uncomplicated
-Firewall]] to blacklist IPs requesting URLs matching a specific pattern.
- 
-Filters from the command line allow to analyze specific periods and
-distinguish traffic generated by self polls and crawlers.
+It is fast. On a ThinkPad P16, a 277M log file is parsed in 15 seconds,
+processing, that is, about 7740 events per second; a 569M log file is parsed in
+50 seconds, that is, about 4700 events per second.
 
-LogSense generates HTML, txt, ufw, and SQLite outputs.
 
-** Rails Report Structure
+** Rails Production Report
 
 #+ATTR_HTML: :width 80%
 [[file:./screenshots/rails-screenshot.png]]
 
-
-** Apache Report Structure
+LogSense understands the Rails *production log* and generates the following
+reports in TXT and HTML:
+
+- Daily Distribution
+- Time Distribution
+- Statuses
+- Statuses by Day
+- Rails Performance
+- Controller and Methods by Device
+- Fatal Events
+- Fatal Events
+- Fatal Events (grouped by type)
+- Job Error
+- Job Errors (grouped) 
+- Browsers
+- Platforms
+- IPs
+- Countries
+- IP per hour
+- Sessions
+
+** Apache/Nginx Report
 
 #+ATTR_HTML: :width 80%
-[[file:./screenshots/apache-screenshot.png]]
-
+[[file:./screenshots/combined_log-screenshot.png]]
+
+LogSense reads the Apache/Nginx *combined log* format and generates the
+following reports in TXT and HTML:
+
+- Time Distribution
+- 20_ and 30_ on HTML pages
+- 20_ and 30_ on other resources
+- 40_ and 50_x on HTML pages
+- 40_ and 50_ on other resources
+- 40_ and 50_x on HTML pages by IP
+- 40_ and 50_ on other resources by IP
+- Statuses
+- Statuses by Day
+- Browsers
+- Platforms
+- IPs
+- Countries
+- IP per hour
+- Combined Platform Data
+- Referers
+- Sessions
 
 ** UFW Report
 
-The output format =ufw= generates directives for Uncomplicated
-Firewall blacklisting IPs requesting URLs matching a given pattern.
+The =ufw= output format generates directives for Uncomplicated Firewall,
+blacklisting IPs requesting URLs matching a given pattern.
 
 We use it to blacklist IPs requesting WordPress login pages on our
 websites... since we don't use WordPress for our websites.
@@ -73,40 +105,55 @@ ufw deny from 185.255.134.18
 ...
 #+end_src
 
-   
-* An important word of warning
-
-[[https://owasp.org/www-community/attacks/Log_Injection][Log poisoning]] is a technique whereby attackers send requests with invalidated
-user input to forge log entries or inject malicious content into the logs.
-
-log_sense sanitizes entries of HTML reports, to try and protect from log
-poisoning.  *Log entries and URLs in SQLite3, however, are not sanitized*:
-they are stored and read from the log.  This is not, in general, an issue,
-unless you use the data from SQLite in environments in which URLs can be
-opened or code executed.
-
-* Motivation
-
-LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] and [[https://umami.is/][Umami]], focusing on
-*privacy*, *data-ownership*, and *simplicity*: no need to install JavaScript
-snippets, no tracking cookies, just plain and simple log analysis.
-
-LogSense is also inspired by *static websites generators*: statistics are
-generated from the command line and accessed as static HTML files.  This
-significantly reduces the attack surface of your web server and installation
-headaches.  We have, for instance, a cron job running on our servers, generating
-statistics at night.  The generated files are then made available on a private
-area on the web.
-
 * Installation
 
   #+begin_src bash
   gem install log_sense
   #+end_src
 
+  If you want to collect information about browsers, platform and devices when
+  generating Rails reports, add the =browser= gem to your bundle and the
+  following code to =application_controller.rb=:
+
+  #+begin_example ruby
+  # Gemfile
+  gem "browser"
+  #+end_example
+
+  #+begin_example ruby
+  # application_controller.rb
+  class ApplicationController < ActionController::Base
+  
+    # [...]
+
+    before_action do |controller|
+      user_agent = request.env['HTTP_USER_AGENT']
+      ip = request.env['REMOTE_ADDR']
+
+      hashed_ip = Digest::SHA256.hexdigest ip
+      b = Browser.new(user_agent)
+      now = DateTime.now
+
+      logger = Rails.logger
+      browser_data = [
+        b.name, b.platform, b.device.name,
+        controller.class.name, controller.action_name,
+        request.format.symbol,
+        hashed_ip,
+        now
+      ]
+
+      browser_data_str = browser_data.map { |x| "\"#{x}\"" }.join(',')
+      logger.info "BrowserInfo: #{browser_data_str}"
+    end
+
+    # [...]
+  end
+  #+end_example
+
 * Usage
 
-  #+begin_src bash :results raw output :wrap example
+  #+begin_src bash :results raw output :wrap example :exports both
   log_sense --help
   #+end_src
 
@@ -131,7 +178,7 @@ area on the web.
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.8.0
+  This is version 2.0.0
 
   Output formats:
 
@@ -146,6 +193,51 @@ log_sense -f apache -i access.log -t txt > access-data.txt
 log_sense -f rails -i production.log -t html -o performance.html
 #+end_example
 
+* Motivation
+
+LogSense focuses on *privacy*, *data-ownership*, and *simplicity*: no need to
+install JavaScript snippets, no tracking cookies, just plain and simple log
+analysis.
+
+LogSense is also inspired by *static websites generators*: statistics are
+generated from the command line and accessed as static HTML files.  This
+significantly reduces the attack surface of your web server and installation
+headaches.  We have a cron job running on our servers, generating statistics at
+night.  The generated files are then made available on a private area on the
+web and rotated monthly.
+
+* An important word of warning on SQLite3 output
+
+[[https://owasp.org/www-community/attacks/Log_Injection][Log poisoning]] is a technique whereby attackers send requests with invalidated
+user input to forge log entries or inject malicious content into the logs.
+
+log_sense sanitizes entries of HTML reports, to try and protect from log
+poisoning.  *Log entries and URLs in SQLite3 tables, however, are not
+sanitized*: they are read and stored from the log as they are.  This is not, in
+general, an issue, unless you use the unsanitized data from SQLite as it is in
+environments where URL can be opened or code executed using the URLs as
+argument.
+
+* Change Log
+
+See the [[file:CHANGELOG.org][CHANGELOG]] file.
+
+* Compatibility
+
+LogSense should run on any system on which a recent version of Ruby
+runs.  We tested it with Ruby 2.6.9 and Ruby 3.0.x, and Ruby 3.3.x
+
+* Author and Contributors
+
+[[https://shair.tech][Shair.Tech]]
+
+* Credits
+
+- HTML reports use [[https://get.foundation/][Zurb Foundation]], [[https://www.datatables.net/][Data Tables]], and [[https://echarts.apache.org/en/index.html][Apache ECharts]]
+- The textual format is compatible with [[https://orgmode.org/][Org Mode]] and can be further processed to
+  any format [[https://orgmode.org/][Org Mode]] can be exported to, including HTML and PDF, with the word
+  of warning in the section above concerning log poisoning.
+
 * Code Structure
 
 The code implements a pipeline, with the following steps:
@@ -164,64 +256,26 @@ The code implements a pipeline, with the following steps:
       building the reports.
   5. *Emitter* generates reports from shaped data using ERB.
 
-The architecture and the structure of the code is far from being nice,
-for historical reason and for a bunch of small differences existing
-between the input and the outputs to be generated.  This usually ends
-up with modifications to the code that have to be replicated in
-different parts of the code and in interferences.
-
-Among the points I would like to address:
-
-- The execution pipeline in the main script has a few exceptions to
-  manage SQLite reading/dumping and ufw report.  A linear structure
-  would be a lot nicer.
-- Two different classes are defined for steps 1, 2, and 4, to manage,
-  respectively, Apache and Rails logs.  These classes inherit from a
-  common ancestor (e.g. ApacheParser and RailsParser both inherit from
-  Parser), but there is still too little code shared.  A nicer
-  approach would be that of identifying a common DB structure and
-  unify the pipeline up to (or including) the generation of
-  reports. There are a bunch of small different things to highlight in
-  reports, which still make this difficult.  For instance, the country
-  report for Apache reports size of TX data, which is not available
-  for Rail reports.
-- Geolocation could become a lot more efficient if performed in
-  SQLite, rather than in Ruby
-- The distinction between Aggregation, Shaping, and Emission is a too
-  fine-grained and it would be nice to be able to cleanly remove one
-  of the steps.
-
-
-* Change Log
-
-See the [[file:CHANGELOG.org][CHANGELOG]] file.
-
-* Compatibility
-
-LogSense should run on any system on which a recent version of Ruby
-runs.  We tested it with Ruby 2.6.9 and Ruby 3.x.x.
-
-Concerning the outputs:
 
-- HTML reports use [[https://get.foundation/][Zurb Foundation]], [[https://www.datatables.net/][Data Tables]], and [[https://vega.github.io/vega-lite/][Vega Light]], which
-  are all downloaded from a CDN
-- The textual format is compatible with [[https://orgmode.org/][Org Mode]] and can be further
-  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to, including HTML
-  and PDF, with the word of warning in the section above. 
+* Todo
 
-* Author and Contributors
-
-[[https://shair.tech][Shair.Tech]]
+See [[todo.org]]
 
 * Known Bugs
 
 We have been running LogSense for quite a few years with no particular issues.
 There are no known bugs; there is an unknown number of unknown bugs.
 
-* License
+You are most welcome to report issues and missing features, using the Issue
+tracker.
+
+* Licenses
 
-Source code distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
+LogSense is distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
 
-Geolocation is made possible by the DB-IP.com IP to City database,
-released under a CC license.
+Geolocation is made possible by [[https://db-ip.com/][dbip]]'s IP to City database, released under a
+CC license.
 
+The world map is distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]] by Pareto
+Softare, [[https://simplemaps.com/][Simplemaps.com]].  It is used in LogSense with some changes to the class
+names and ids.

data/exe/log_sense

@@ -114,7 +114,7 @@ elsif @options[:output_format] == "ufw"
   }
   ips_and_urls.each do |ip, urls|
     puts "# #{urls[0..10].uniq.join(' ')}"
-    puts "ufw deny from #{ip}"
+    puts "ufw insert 1 deny from #{ip}"
     puts
   end
 
@@ -132,6 +132,7 @@ else
 
     warn "Grouping IPs by country ..." if @options[:verbose]
     country_col = geolocated_data[0].size - 1
+    @data[:ips] = geolocated_data
     @data[:countries] = geolocated_data.group_by { |x| x[country_col] }
   elsif @options[:geolocation] && @data[:ips].size == 0
     warn "Skipping geolocation: no IP found" if @options[:verbose]

data/lib/log_sense/aggregator.rb

@@ -78,7 +78,8 @@ module LogSense
         extra_cols = ""
       end
 
-      @ips = @db.execute %(SELECT ip, count(ip) #{extra_cols} from #{@table}
+      @ips = @db.execute %(SELECT ip, count(ip) #{extra_cols}
+                                  from #{@table}
                                   where #{filter}
                                   group by ip                 
                                   order by count(ip) desc     
@@ -169,7 +170,7 @@ module LogSense
     # name is used to give the name to the column with formatted time
     def ip_by_time_query(name, format_string)
       %(SELECT ip,
-               strftime("%H", #{@date_field}) as #{name},
+               strftime('#{format_string}', #{@date_field}) as #{name},
                count(#{@url_field}) from #{@table}
                where #{filter} and ip != "" and
                #{@url_field} != "" and

data/lib/log_sense/apache_aggregator.rb

@@ -112,7 +112,7 @@ module LogSense
               count(path),
               count(distinct(unique_visitor)),
               #{human_readable_size}, status from LogLine
-       where #{result} and #{type} and #{filter}
+       where #{filter} and #{result} and #{type}
        group by path
        order by count(path) desc
        limit #{@options[:limit]}"