log_sense 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6c9423e2199ce12c9ebf30af86a3b9f2ee2edf63de1555f12dbe09c8b798179e
+  data.tar.gz: 59ccd77c5a7d65943705f5b15a4ec8427f12c63ae9b1468b16e46bfe36cd53ce
+SHA512:
+  metadata.gz: 1f4dadc41047040dc2e36a91b55e2f9d5d759556d8e5d8a77557d3088eda873f8ac4dc3968f7d0617b6aefa5b5507fbe1008f1baaf5aa67ab430bedb0b858bf2
+  data.tar.gz: 699e717c1118196b7cf90bcee9774f85b3e298ce15e1e1bceb0715a7bcd5094c9c933ee416663a0a28eb1f3b3bc084fb15060a6a6fb3d8bdfb19b2c44d61bf60

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+*~

data/CHANGELOG.org

@@ -0,0 +1,27 @@
+#+TITLE: ChangeLog
+#+AUTHOR: Adolfo Villafiorita
+#+STARTUP: showall
+
+* Unreleased
+
+This changes are in the repository but not yet released to Rubygems.
+
+** New Functions and Changes
+
+** Fixes
+
+** Documentation
+
+** Code
+
+
+* Version 1.0.0
+
+** New Functions and Changes
+
+** Fixes
+
+** Documentation
+
+** Code
+

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in apache_log_report.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    log_sense (1.2.0)
+      apache_log-parser
+      browser
+      ipaddr
+      iso_country_codes
+      sqlite3
+      terminal-table
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    apache_log-parser (3.1.2)
+    browser (5.3.1)
+    byebug (11.1.3)
+    ipaddr (1.2.3)
+    iso_country_codes (0.7.8)
+    minitest (5.14.4)
+    rake (12.3.3)
+    sqlite3 (1.4.2)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  log_sense!
+  minitest
+  rake (~> 12.0)
+
+BUNDLED WITH
+   2.2.29

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Adolfo Villafiorita
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.org

@@ -0,0 +1,114 @@
+#+TITLE: README
+#+AUTHOR: Adolfo Villafiorita
+#+STARTUP: showall
+
+* Introduction
+
+LogSense generates reports and statistics from Apache web logs in the
+=combined= format and from Rails logs.  Written in Ruby, it runs from
+the command line, it is fast, and it can be installed on any system
+which supports Ruby.
+
+LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]]
+and [[https://umami.is/][Umami]], focusing on privacy and data-ownership: the data
+generated by LogSense is stored on your computer and owned by
+you (like it should be).
+
+LogSense is also inspired by static websites generators:
+statistics are generated from the command line and accessed as static
+HTML files.  By generating static resources, LogSense
+significantly reduces the attack surface of your webserver and
+installation headaches.
+
+We have, for instance, a cron job running on our servers, generating
+statistics at night.  The generated files are then made available on a
+private area on the web.
+
+Statistics are generated from Apache log formats in the =combined=
+format and from Rails logs.  Reports are tailored, but not limited, to
+web servers serving static websites.  No need to install Java Script
+code on your websites, no cookies installed, no user tracking.
+
+LogSense reports the following data:
+
+- Visitors, hits, unique visitors, bandwidth used
+- Most accessed HTML pages
+- Most accessed resources  
+- Response statuses
+- Referers
+- OS, browsers, and devices
+- IP Country location, thanks to the DPIP lite country DB
+- Streaks: resources accessed by a given IP over time
+- Potential attacks: access to resources which are not meant to be
+  served by a web server serving static websites
+- Performance of Rails requests
+ 
+Filters from the command line allow to analyze specific periods and
+distinguish traffic generated by self polls and crawlers.
+
+LogSense generates HTML, txt (Org Mode), and SQLite outputs.
+
+* Installation
+
+  #+begin_src bash
+  gem install log_sense
+  #+end_src
+
+* Usage
+
+  #+begin_src bash :results raw output :wrap example
+  log_sense --help
+  #+end_src
+
+  #+RESULTS:
+  #+begin_example
+  Usage: apache_log_report [options] [logfile]
+      -l, --limit=N                    Number of entries to show (defaults to 30)
+      -b, --begin=DATE                 Consider entries after or on DATE
+      -e, --end=DATE                   Consider entries before or on DATE
+      -i, --ignore-crawlers            Ignore crawlers
+      -p, --ignore-selfpoll            Ignore apaches self poll entries (from ::1)
+          --only-crawlers              Perform analysis on crawlers only
+      -u, --prefix=PREFIX              Prefix to add to all plots (used to run multiple analyses in the same dir)
+      -w, --suffix=SUFFIX              Suffix to add to all plots (used to run multiple analyses in the same dir)
+      -c, --code-export=WHAT           Control :export directive in Org Mode code blocks (code, results, *both*, none)
+      -f, --format=FORMAT              Output format: html, org, sqlite. Defaults to org mode
+      -v, --version                    Prints version information
+      -h, --help                       Prints this help
+  This is version 1.1.6
+  #+end_example
+
+* Change Log
+
+See the [[file:CHANGELOG.org][CHANGELOG]] file.
+
+* Compatibility
+
+LogSense should run on any system on which Ruby runs.
+
+Concerning the outputs:
+
+- The HTML report uses [[https://picturepan2.github.io/spectre/][Spectre.css]] and (will use) [[https://vega.github.io/vega-lite/][Vega Light]], which
+  are downloaded from a CDN
+- The textual format is compatible with Org Mode and can be further
+  processed to any format Org Mode can be exported to (including HTML
+  and PDF),
+
+* Author and Contributors
+
+[[http://ict4g.net/adolfo][Adolfo Villafiorita]].
+
+* Known Bugs
+
+Some known bugs and an unknown number of unknown bugs.
+
+(See the open issues for the known bugs.)
+
+* License
+
+Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].
+
+Geolocation is made possible by the DB-IP.com IP to City database, released under
+a CC license.
+
+

data/Rakefile

@@ -0,0 +1,15 @@
+require "bundler/gem_tasks"
+task :default => :spec
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+require_relative './lib/log_sense/ip_locator.rb'
+
+desc "Convert Geolocation DB to sqlite"
+task :dbip_to_sqlite3, [:filename] do |tasks, args|
+  filename = args[:filename]
+  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+end

data/alr-styles.css

@@ -0,0 +1,61 @@
+nav {
+    position: fixed;
+}
+
+section {
+    margin-left: 250px;
+}
+
+article {
+    margin-top: 1rem;
+}
+
+h1, h2 {
+    color: #222222 !important;
+}
+
+/*
+table {
+    border: 1px solid #222222;
+    border-collapse: collapse;
+}
+*/
+
+table th {
+    background: #444444;
+    color: white;
+}
+
+.ip {
+    vertical-align: top;
+}
+ 
+.summary th {
+    text-align: left;
+}
+
+.summary td {
+    text-align: right;
+}
+
+.hits, .visits, .size, .visitors, .count {
+    text-align: right;
+    font-weight: bold;
+}
+
+.referers .size {
+    width: 20%;
+    font-weight: bold;
+}
+
+.command-invocation, .log-structure, .performance {
+    width: 60%;
+}
+ 
+.command-invocation th, .log-structure th, .performance th {
+    text-align: left;
+}
+
+.log-structure td, .performance td {
+    text-align: right;
+}

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "log_sense"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/log_sense

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'log_sense.rb'
+
+#
+# Parse Command Line Arguments
+#
+
+# better be here... OptionsParser consumes ARGV
+@command_line = ARGV.join(" ")
+@options     = LogSense::OptionsParser.parse ARGV
+@input_file  = @options[:input_file]
+@output_file = @options[:output_file]
+
+if not @input_file
+  puts "Error: no input file specified."
+  exit
+end
+
+if not File.exist? @input_file
+  puts "Error: input file '#{@input_file}' does not exist"
+  exit 1
+end
+
+#
+# Parse Log and Track Statistics
+#
+
+@started_at = Time.now
+
+case @options[:input_format]
+when 'apache'
+  parser_klass = LogSense::ApacheLogParser
+  cruncher_klass = LogSense::ApacheDataCruncher
+when 'rails'
+  parser_klass = LogSense::RailsLogParser
+  cruncher_klass = LogSense::RailsDataCruncher
+end
+
+@db = parser_klass.parse @input_file
+
+if @options[:output_format]  == "sqlite"
+  ddb = SQLite3::Database.new(@output_file || "db.sqlite3")
+  b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
+  b.step(-1) #=> DONE
+  b.finish
+else
+  @data = cruncher_klass.crunch @db, @options
+  @data = LogSense::IpLocator.geolocate @data
+
+  @ended_at = Time.now
+  @duration = @ended_at - @started_at
+
+  @data = @data.merge({
+                        command: @command_line,
+                        log_file: @input_file,
+                        started_at: @started_at,
+                        ended_at: @ended_at,
+                        duration: @duration
+                      })
+
+  #
+  # Emit Output
+  #
+  puts LogSense::Emitter.emit @data, @options
+end

data/lib/log_sense/apache_data_cruncher.rb

@@ -0,0 +1,131 @@
+module LogSense
+  module ApacheDataCruncher
+    #
+    # take a sqlite3 database and analyze data
+    #
+    # @ variables are automatically put in the returned data
+    #
+
+    def self.crunch db, options = { limit: 30 }
+      first_day_s = db.execute "SELECT datetime from LogLine order by datetime limit 1"
+      last_day_s  = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
+
+      # make first and last day into dates or nil
+      @first_day = first_day_s.empty? ? nil : Date.parse(first_day_s[0][0])
+      @last_day = last_day_s.empty? ? nil : Date.parse(last_day_s[0][0])
+
+      @total_days = 0
+      if @first_day and @last_day
+        @total_days = (@last_day - @first_day).to_i
+      end
+
+      @log_size       = db.execute "SELECT count(datetime) from LogLine"
+      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
+
+      @first_day_requested = options[:from_date]
+      @last_day_requested = options[:to_date]
+
+      @first_day_in_analysis = date_intersect options[:from_date], @first_day, :max
+      @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
+
+      @total_days_in_analysis = 0
+      if @first_day_in_analysis and @last_day_in_analysis
+        @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
+      end
+
+      #
+      # generate the where clause corresponding to the command line options to filter data
+      #
+      filter = [
+        (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
+        (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
+        (options[:only_crawlers] ? "bot == 1" : nil),
+        (options[:ignore_crawlers] ? "bot == 0" : nil),
+        (options[:no_selfpolls] ? "ip != '::1'" : nil),
+        "true"
+      ].compact.join " and "
+
+      mega = 1024 * 1024
+      giga = mega * 1024
+      tera = giga * 1024
+      
+      # in alternative to sum(size)
+      human_readable_size = <<-EOS
+      CASE 
+      WHEN sum(size) <  1024 THEN sum(size) || ' B' 
+      WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
+      WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
+      WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
+      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
+      END AS size
+      EOS
+
+      human_readable_day = <<-EOS
+        case cast (strftime('%w', datetime) as integer)
+          when 0 then 'Sunday'
+          when 1 then 'Monday'
+          when 2 then 'Tuesday'
+          when 3 then 'Wednesday'
+          when 4 then 'Thursday'
+          when 5 then 'Friday'
+          else 'Saturday'
+        end as dow
+      EOS
+
+      @total_hits = db.execute "SELECT count(datetime) from LogLine where #{filter}"
+      @total_unique_visitors = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{filter}"
+      @total_size = db.execute "SELECT #{human_readable_size} from LogLine where #{filter}"
+
+      @daily_distribution = db.execute "SELECT date(datetime), #{human_readable_day}, count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
+      @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"
+      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+      @reasonable_requests_exts = [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].map {  |x|
+        "extension != '#{x}'"
+      }.join " and "
+
+      @attacks = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} and (#{@reasonable_requests_exts}) group by path order by count(path) desc  limit #{options[:limit]}"
+      @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
+
+      @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"
+      @by_day_3xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '3' and #{filter} group by date(datetime)"
+      @by_day_2xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '2' and #{filter} group by date(datetime)"
+
+      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
+        [x[0], x[1].map { |y| y[1] }].flatten
+      }
+
+      @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
+      @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
+      @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
+      
+      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
+
+      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      data = {}
+
+      self.instance_variables.each do |variable|
+        var_as_symbol = variable.to_s[1..-1].to_sym
+        data[var_as_symbol] = eval(variable.to_s)
+      end
+      data
+    end
+
+    private
+
+    def self.date_intersect date1, date2, method
+      if date1 and date2
+        [date1, date2].send(method)
+      elsif date1
+        date1
+      else
+        date2
+      end
+    end
+  end
+end
+

data/lib/log_sense/apache_log_parser.rb

@@ -0,0 +1,87 @@
+require 'apache_log/parser'
+require 'sqlite3'
+require 'browser'
+
+module LogSense
+  module ApacheLogParser
+    #
+    # parse an Apache log file and return a SQLite3 DB
+    #
+
+    def self.parse filename, options = {}
+      content = filename ? File.readlines(filename) : ARGF.readlines
+
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE IF NOT EXISTS LogLine(
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      datetime TEXT,
+      ip TEXT,
+      user TEXT,
+      unique_visitor TEXT,
+      method TEXT,
+      path TEXT,
+      extension TEXT,
+      status TEXT,
+      size INTEGER,
+      referer TEXT,
+      user_agent TEXT,
+      bot INTEGER,
+      browser TEXT,
+      browser_version TEXT,
+      platform TEXT,
+      platform_version TEXT)"
+      
+      ins = db.prepare('insert into LogLine (
+                datetime, 
+                ip,
+                user,
+                unique_visitor,
+                method,
+                path, 
+                extension,
+                status,
+                size,
+                referer,
+                user_agent,
+                bot,
+                browser,
+                browser_version,
+                platform,
+                platform_version)
+              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+
+      parser = ApacheLog::Parser.new(options[:format] || 'combined')
+      
+      content.each do |line|
+        begin
+          hash = parser.parse line
+
+          ua = Browser.new(hash[:user_agent], accept_language: "en-us")
+          ins.execute(
+            hash[:datetime].iso8601,
+            hash[:remote_host],
+            hash[:user],
+            hash[:datetime].strftime("%Y-%m-%d") + " " + hash[:remote_host] + " " + hash[:user_agent],
+            hash[:request][:method],
+            hash[:request][:path],
+            (hash[:request][:path] ? File.extname(hash[:request][:path]) : ""),
+            hash[:status],
+            hash[:size].to_i,
+            hash[:referer],
+            hash[:user_agent],
+            ua.bot? ? 1 : 0,
+            (ua.name || ""),
+            (ua.version || ""),
+            (ua.platform.name || ""),
+            (ua.platform.version || "")
+          )
+        rescue
+          STDERR.puts "Apache Log parser error: could not parse #{line}"
+        end
+      end
+      
+      db
+    end
+
+  end
+end

data/lib/log_sense/emitter.rb

@@ -0,0 +1,49 @@
+require 'terminal-table'
+require 'erb'
+require 'ostruct'
+
+module LogSense
+  module Emitter
+
+    #
+    # Emit Data
+    #
+    def self.emit data = {}, options = {}
+      @input_format = options[:input_format] || "apache"
+      @output_format = options[:output_format] || "html"
+
+      # for the ERB binding
+      @data = data
+      @options = options
+
+      # determine the main template to read
+      @template = File.join(File.dirname(__FILE__), "templates", "#{@input_format}.#{@output_format}.erb")
+      erb_template = File.read @template
+      
+      output = ERB.new(erb_template).result(binding)
+
+      if options[:output_file]
+        file = File.open options[:output_file], "w"
+        file.write output
+        file.close
+      else
+        puts output
+      end
+    end
+
+    private
+
+    def self.output_txt_table name, headings, rows
+      name = "#+NAME: #{name}"
+      table = Terminal::Table.new headings: headings, rows: rows, style: { border_x: "-", border_i: "|" }
+      name + "\n" + table.to_s
+    end
+
+    def self.render(template, vars)
+      @template = File.join(File.dirname(__FILE__), "templates", "_#{template}.html.erb")
+      erb_template = File.read @template
+      ERB.new(erb_template).result(OpenStruct.new(vars).instance_eval { binding })
+    end
+
+  end
+end