log_sense 1.6.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a915af429fe2492f17bc767c86767e38d37a674af6206fb3b2c0a76e4cad620
-  data.tar.gz: bf4cc3a1e438b752c9c99fee5f91c85abd38de95dac947c2382e2f9825b51467
+  metadata.gz: 126257c949e11f090cc1928a1458572529f71c44a5c20baae35861241dfa7b7b
+  data.tar.gz: 33a1ee650598a90ca9adb6a6c2795746dfe8a7735414b22b930d3806b205b318
 SHA512:
-  metadata.gz: 5612ef5474aa397132527588d289d9d1eba4f8954b92553e32fd5b856f2bd5441ba09d89d1bf12a0f220c602dcd2e73de2d6e360b6177b162d57adc2726442c9
-  data.tar.gz: c3b546cc177a3364b1b513f4274c1521aa1669bb17b142eb453ba73251f1e3458e7b9d1703b692ef275a474821ce7375e387155f63e3e7d5d7c9c42e1c50a150
+  metadata.gz: f135c70480994434dea0b5ff11bac5b1239d071ae38afeda7dd3672a43e127bba06136d550f29844c60dd2d2640a87c57f7fda683240512d1fe37794d124a433
+  data.tar.gz: 65abbe86864aba7d9e6499e6f3ec4a0ef2facf00505ea8611eec3b44279772641bf3a4374045acac0892c8bde19ce37c74a88501fa7905342b1e30032f2b50b4

data/CHANGELOG.org

@@ -2,6 +2,18 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.7.0
+
+- [User] Fixes a bug with the geolocator
+- [User] Fixes a bug causing a crash when no country was found by the geolocator
+- [User] Fixes bugs related to corner cases (empty logs, wrong parser for log,
+  empty geolocation data)
+- [User] Updated DB-IP country file to Jun 2024 version.
+- [User] Refreshed the style a bit, removed Fira Sans and updated versions of
+  CSS and JS frameworks
+- [Code] Move options and some code in their own dir
+- [Code] Add rendering view parsing (useful in development; no views yet)
+
 * 1.6.1
 
 - Country DB now stores country name.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.5.3)
+    log_sense (1.7.0)
       browser
       ipaddr
       iso_country_codes
@@ -12,24 +12,30 @@ GEM
   remote: https://rubygems.org/
   specs:
     browser (5.3.1)
-    debug (1.6.2)
-      irb (>= 1.3.6)
-      reline (>= 0.3.1)
-    io-console (0.5.11)
-    ipaddr (1.2.5)
-    irb (1.4.1)
-      reline (>= 0.3.0)
+    debug (1.9.2)
+      irb (~> 1.10)
+      reline (>= 0.3.8)
+    io-console (0.7.2)
+    ipaddr (1.2.6)
+    irb (1.13.1)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
     iso_country_codes (0.7.8)
-    mini_portile2 (2.8.0)
-    minitest (5.15.0)
+    mini_portile2 (2.8.7)
+    minitest (5.23.1)
+    psych (5.1.2)
+      stringio
     rake (12.3.3)
-    reline (0.3.1)
+    rdoc (6.7.0)
+      psych (>= 4.0.0)
+    reline (0.5.8)
       io-console (~> 0.5)
-    sqlite3 (1.5.4)
+    sqlite3 (2.0.2)
       mini_portile2 (~> 2.8.0)
+    stringio (3.1.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.3.0)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   ruby
@@ -41,4 +47,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.3.3
+   2.5.3

data/Rakefile

@@ -8,9 +8,9 @@ end
 
 require_relative './lib/log_sense/ip_locator.rb'
 
-desc "Convert Geolocation DB to sqlite"
+desc "Convert Geolocation DB to sqlite (arg YYYY_MM or filename)"
 task :dbip, [:filename] do |tasks, args|
-  filename_or_yyyy_mm = args[:filename]
+  filename_or_yyyy_mm = args[:filename] || ""
 
   filename = if /\d{4}-\d{2}/.match(filename_or_yyyy_mm)
                "ip_locations/dbip-country-lite-#{filename_or_yyyy_mm}.csv"
@@ -18,13 +18,15 @@ task :dbip, [:filename] do |tasks, args|
                filename_or_yyyy_mm
              end
 
-  # if the filename has a .gz extension or a gzipped version of the file 
-  # exists, gunzip it
+  # if the filename passed as argument has a .gz extension or a gzipped version
+  # of the file passed as argument exists, gunzip it
   if File.extname(filename) == ".gz" || File.exist?("#{filename}.gz")
     system "gunzip #{filename}.gz"
   end
 
-  if !File.exist? filename
+  if File.exist? filename
+    LogSense::IpLocator::dbip_to_sqlite filename
+  else
     puts <<-EOS
 Error. Could not find: #{filename}
 
@@ -37,13 +39,10 @@ I see the following files:
 3. Relaunch with YYYY-MM (will build: dbip-country-lite-YYYY-MM.csv)
    or with filename.
 
-Remark. If the filename has the extension .gz or if the
-filename does not exist, but a file with the same name and .gz extension
-exists, it is gunzipped first
+Remark. If the filename has the extension .gz or if the filename does not exist,
+but a file with the same name and .gz extension exists, it is gunzipped first
     EOS
 
     exit
-  else
-    LogSense::IpLocator::dbip_to_sqlite filename
   end
 end

data/exe/log_sense

@@ -9,7 +9,7 @@ require "sqlite3"
 
 # this better be here... OptionsParser consumes ARGV
 @command_line = ARGV.join(" ")
-@options = LogSense::OptionsParser.parse ARGV
+@options = LogSense::Options::Parser.parse ARGV
 @input_filenames = @options[:input_filenames] + ARGV
 @output_filename = @options[:output_filename]
 
@@ -20,33 +20,36 @@ require "sqlite3"
 #
 # Check input files
 #
-@non_existing = @input_filenames.reject { |x| File.exist?(x) }
 
+@non_existing = @input_filenames.reject { |x| File.exist?(x) }
 if @non_existing.any?
   warn "Error: some input file(s) \"#{@non_existing.join(", ")}\" do not exist"
   exit 1
 end
 
-#
-# Special condition: sqlite3 requires a single file as input
-#
-if @input_filenames.size > 0 &&
-   File.extname(@input_filenames.first) == "sqlite3" &&
-   @input_filenames.size > 1
-  warn "Error: you can pass only one sqlite3 file as input"
+@sqlite3_files = @input_filenames.select { |x| File.extname(x).include?("sqlite") }
+if @sqlite3_files.any? && @input_filenames.size != 1
+  warn "Error: when passing an SQLite3 DB, this has to be the only input file"
   exit 1
 end
 
+#
+# Check output files
+#
+
+# Nothing to be done, here, since we output to STDOUT if no output filename is
+# specified
+
 #
 # Supported input/output chains
 #
 iformat = @options[:input_format]
 oformat = @options[:output_format]
 
-if !LogSense::OptionsChecker::compatible?(iformat, oformat)
+if !LogSense::Options::Checker.compatible?(iformat, oformat)
   warn "Error: don't know how to make #{iformat} into #{oformat}."
   warn "Possible transformation chains:"
-  warn LogSense::OptionsChecker.chains_to_s
+  warn LogSense::Options::Checker.chains_to_s
   exit 1
 end
 
@@ -56,8 +59,11 @@ end
 
 @started_at = Time.now
 
-if @input_filenames.size > 0 &&
-   File.extname(@input_filenames.first) == ".sqlite3"
+#
+# Input
+#
+
+if @input_filenames.size > 0 && File.extname(@input_filenames.first) == ".sqlite3"
   warn "Reading SQLite3 DB ..." if @options[:verbose]
   @db = SQLite3::Database.open @input_filenames.first
 else
@@ -67,12 +73,19 @@ else
                  else
                    @input_filenames.map { |fname| File.open(fname, "r") }
                  end
-  class_name = "LogSense::#{@options[:input_format].capitalize}LogParser"
+
+  class_name = "LogSense::#{@options[:input_format].capitalize}::LogParser"
   parser_class = Object.const_get class_name
   parser = parser_class.new
   @db = parser.parse @input_files
 end
 
+#
+# Output
+# 
+
+# TODO this code could benefit from some classes abstracting the work a bit
+
 if @options[:output_format] == "sqlite3"
   warn "Saving SQLite3 DB ..." if @options[:verbose]
 
@@ -83,7 +96,7 @@ if @options[:output_format] == "sqlite3"
 
   exit 0
 elsif @options[:output_format] == "ufw"
-  pattern = @options[:pattern] || "php"
+  pattern = @options[:pattern]
 
   if @options[:input_format] == "rails"
     query = "select distinct event.ip,event.url
@@ -113,14 +126,18 @@ else
   aggr = aggr_class.new(@db, @options)
   @data = aggr.aggregate
 
-  if @options[:geolocation]
+  if @options[:geolocation] && @data[:ips].size != 0
     warn "Geolocating ..." if @options[:verbose]
-    @data = LogSense::IpLocator.geolocate @data
+    geolocated_data = LogSense::IpLocator.geolocate @data
 
     warn "Grouping IPs by country ..." if @options[:verbose]
-    country_col = @data[:ips][0].size - 1
-    @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+    country_col = geolocated_data[0].size - 1
+    @data[:countries] = geolocated_data.group_by { |x| x[country_col] }
+  elsif @options[:geolocation] && @data[:ips].size == 0
+    warn "Skipping geolocation: no IP found" if @options[:verbose]
+    @data[:countries] = {}
   else
+    warn "Skipping geolocation." if @options[:verbose]
     @data[:countries] = {}
   end
 

data/lib/log_sense/apache/log_line_parser.rb

@@ -0,0 +1,59 @@
+module LogSense
+  module Apache
+    # parses a log line and returns a hash
+    # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
+    #
+    # %h: IP
+    # %l: ident or -
+    # %u: userid or -
+    # %t: [10/Oct/2000:13:55:36 -0700]
+    #   day = 2*digit
+    #   month = 3*letter
+    #   year = 4*digit
+    #   hour = 2*digit
+    #   minute = 2*digit
+    #   second = 2*digit
+    #   zone = (`+' | `-') 4*digit
+    # %r: GET /apache_pb.gif HTTP/1.0
+    # %{User-agent}: "
+    #
+    # Example
+    # 116.179.32.16 - - [19/Dec/2021:22:35:11 +0100] "GET / HTTP/1.1" 200 135 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
+    #
+    class LogLineParser
+      DAY = /[0-9]{2}/
+      MONTH = /[A-Za-z]{3}/
+      YEAR = /[0-9]{4}/
+      TIMEC = /[0-9]{2}/
+      TIMEZONE = /(\+|-)[0-9]{4}/
+
+      IP = /(?<ip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|::1|unknown)/
+      IDENT = /(?<ident>[^ ]+|-)/
+      USERID = /(?<userid>[^ ]+|-)/
+
+      TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
+
+      HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
+      WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
+      OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
+      METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
+      PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
+      URL = /(?<url>[^ ]+)/
+      REFERER = /(?<referer>[^"]*)/
+      RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
+      SIZE = /(?<size>[0-9]+|-)/
+      USER_AGENT = /(?<user_agent>[^"]*)/
+
+      attr_reader :format
+
+      def initialize
+        @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
+      end
+
+      def parse(line)
+        @format.match(line) ||
+          raise("Apache LogLine Parser Error: Could not parse #{line}")
+      end
+    end
+  end
+end

data/lib/log_sense/apache/log_parser.rb

@@ -0,0 +1,101 @@
+require "sqlite3"
+require "browser"
+require_relative "log_line_parser"
+
+module LogSense
+  module Apache
+    #
+    # parse an Apache log file and return a SQLite3 DB
+    #
+    class LogParser
+      def parse(streams, options = {})
+        db = SQLite3::Database.new ":memory:"
+
+        db.execute "CREATE TABLE IF NOT EXISTS LogLine(
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      datetime TEXT,
+      ip TEXT,
+      user TEXT,
+      unique_visitor TEXT,
+      method TEXT,
+      path TEXT,
+      extension TEXT,
+      status TEXT,
+      size INTEGER,
+      referer TEXT,
+      user_agent TEXT,
+      bot INTEGER,
+      browser TEXT,
+      browser_version TEXT,
+      platform TEXT,
+      platform_version TEXT,
+      source_file TEXT,
+      line_number INTEGER
+      )"
+
+        ins = db.prepare("insert into LogLine (
+                datetime,
+                ip,
+                user,
+                unique_visitor,
+                method,
+                path,
+                extension,
+                status,
+                size,
+                referer,
+                user_agent,
+                bot,
+                browser,
+                browser_version,
+                platform,
+                platform_version,
+                source_file,
+                line_number
+                )
+              values (#{Array.new(18, '?').join(', ')})")
+
+        parser = LogLineParser.new
+
+        streams.each do |stream|
+          stream.readlines.each_with_index do |line, line_number|
+            begin
+              hash = parser.parse line
+              ua = Browser.new(hash[:user_agent], accept_language: 'en-us')
+              ins.execute(
+                DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
+                hash[:ip],
+                hash[:userid],
+                unique_visitor_id(hash),
+                hash[:method],
+                hash[:url],
+                (hash[:url] ? File.extname(hash[:url]) : ''),
+                hash[:status],
+                hash[:size].to_i,
+                hash[:referer],
+                hash[:user_agent],
+                ua.bot? ? 1 : 0,
+                (ua.name || ''),
+                (ua.version || ''),
+                (ua.platform.name || ''),
+                (ua.platform.version || ''),
+                stream == $stdin ? "stdin" : stream.path,
+                line_number
+              )
+            rescue StandardError => e
+              warn e.message
+            end
+          end
+        end
+
+        db
+      end
+
+      private
+
+      def unique_visitor_id hash
+        "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
+      end
+    end
+  end
+end

data/lib/log_sense/emitter.rb

@@ -10,18 +10,18 @@ module LogSense
   #
   class Emitter
     CDN_CSS = [
-      "https://cdnjs.cloudflare.com/ajax/libs/foundicons/3.0.0/foundation-icons.min.css",
-      "https://cdn.jsdelivr.net/npm/foundation-sites@6.7.5/dist/css/foundation.min.css",
-      "https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.css"
+      # "https://cdnjs.cloudflare.com/ajax/libs/foundicons/3.0.0/foundation-icons.min.css",
+      "https://cdn.jsdelivr.net/npm/foundation-sites@6.8.1/dist/css/foundation.min.css",
+      "https://cdn.datatables.net/v/zf/dt-2.0.8/datatables.min.css"
     ].freeze
 
     CDN_JS = [
-      "https://code.jquery.com/jquery-3.6.2.min.js",
-      "https://cdn.datatables.net/v/zf/dt-1.13.1/datatables.min.js",
-      "https://cdn.jsdelivr.net/npm/foundation-sites@6.7.5/dist/js/foundation.min.js",
-      "https://cdn.jsdelivr.net/npm/vega@5.22.1",
-      "https://cdn.jsdelivr.net/npm/vega-lite@5.6.0",
-      "https://cdn.jsdelivr.net/npm/vega-embed@6.21.0"
+      "https://code.jquery.com/jquery-3.7.1.min.js",
+      "https://cdn.datatables.net/v/zf/dt-2.0.8/datatables.min.js",
+      "https://cdn.jsdelivr.net/npm/foundation-sites@6.8.1/dist/js/foundation.min.js",
+      "https://cdn.jsdelivr.net/npm/vega@5.28.0",
+      "https://cdn.jsdelivr.net/npm/vega-lite@5.18.1",
+      "https://cdn.jsdelivr.net/npm/vega-embed@6.25.0"
     ].freeze
 
     def self.emit(reports = {}, data = {}, options = {})
@@ -66,21 +66,27 @@ module LogSense
       end
     end
 
-    def self.escape_javascript(string)
-      js_escape_map = {
-        #"&" => "&",
-        #"%" => "%",
-        "<" => "&lt;",
-        "\\" => "&bsol;",
-        '"' => ' \\"',
-        "'" => " \\'",
-        "`" => " \\`",
-        "$" => " \\$"
-      }
-      js_escape_map.each do |match, replace|
-        string = string.gsub(match, replace)
+    # taken from Ruby on Rails
+    JS_ESCAPE_MAP = {
+      "\\" => "\\\\",
+      "</" => '<\/',
+      "\r\n" => '\n',
+      "\n" => '\n',
+      "\r" => '\n',
+      '"' => '\\"',
+      "'" => "\\'",
+      "`" => "\\`",
+      "$" => "\\$"
+    }
+
+    # taken from Ruby on Rails
+    def self.escape_javascript(javascript)
+      javascript = javascript.to_s
+      if javascript.empty?
+        ""
+      else
+        javascript.gsub(/(\\|<\/|\r\n|\342\200\250|\342\200\251|[\n\r"']|[`]|[$])/u, JS_ESCAPE_MAP)
       end
-      string
     end
 
     def self.slugify(string)

data/lib/log_sense/ip_locator.rb

@@ -46,7 +46,7 @@ module LogSense
     end
 
     def merge(parser_db)
-      ipdb = Sqlite3::Database.open DB_FILE
+      Sqlite3::Database.open DB_FILE
       parser_db
     end
 
@@ -75,11 +75,14 @@ module LogSense
     def self.geolocate(data)
       @location_db = IpLocator.load_db
 
-      data[:ips].each do |line|
-        country_code = IpLocator.locate_ip line[0], @location_db
-        line << country_code
+      data[:ips].map do |line|
+        begin
+          country_code = IpLocator.locate_ip line[0], @location_db
+          line + [country_code]
+        rescue
+          line + ["INVALID IP"]
+        end
       end
-      data
     end
   end
 end

data/lib/log_sense/options/checker.rb

@@ -0,0 +1,26 @@
+module LogSense
+  #
+  # Check options and return appropriate error if
+  # combinations of command arguments are wrong
+  #
+  module Options
+    module Checker
+      SUPPORTED_CHAINS = {
+        rails: %i[txt html sqlite3 ufw],
+        apache: %i[txt html sqlite3 ufw]
+      }.freeze
+
+      def self.compatible?(iformat, oformat)
+        (SUPPORTED_CHAINS[iformat.to_sym] || []).include? oformat.to_sym
+      end
+
+      def self.chains_to_s
+        string = ""
+        SUPPORTED_CHAINS.each do |iformat, oformat|
+          string << "- #{iformat}: #{oformat.join(", ")}\n"
+        end
+        string
+      end
+    end
+  end
+end