log_sense 1.5.2 → 1.6.0

data/lib/log_sense/ip_locator.rb

@@ -1,37 +1,47 @@
-require 'csv'
-require 'sqlite3'
-require 'ipaddr'
-require 'iso_country_codes'
+require "csv"
+require "sqlite3"
+require "ipaddr"
+require "iso_country_codes"
 
 module LogSense
   #
   # Populate table of IP Locations from dbip-country-lite
   #
   module IpLocator
-    DB_FILE = File.join(File.dirname(__FILE__), '..', '..', 'ip_locations', 'dbip-country-lite.sqlite3')
+    DB_FILE = File.join(File.dirname(__FILE__), "..", "..", "ip_locations", "dbip-country-lite.sqlite3")
 
     def self.dbip_to_sqlite(db_location)
-      db = SQLite3::Database.new ':memory:'
-      db.execute 'CREATE TABLE ip_location (
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE ip_location (
         from_ip_n INTEGER,
         from_ip TEXT,
         to_ip TEXT,
         country_code TEXT
-      )'
+      )"
 
-      ins = db.prepare 'INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)'
+      ins = db.prepare "INSERT INTO ip_location(
+                               from_ip_n, from_ip, to_ip, country_code)
+                               values (?, ?, ?, ?)"
       CSV.foreach(db_location) do |row|
+        # skip ip v6 addresses
+        next if row[0].include?(":")
+
         ip = IPAddr.new row[0]
         ins.execute(ip.to_i, row[0], row[1], row[2])
       end
 
       # persist to file
       ddb = SQLite3::Database.new(DB_FILE)
-      b = SQLite3::Backup.new(ddb, 'main', db, 'main')
+      b = SQLite3::Backup.new(ddb, "main", db, "main")
       b.step(-1) #=> DONE
       b.finish
     end
 
+    def merge(parser_db)
+      ipdb = Sqlite3::Database.open DB_FILE
+      parser_db
+    end
+
     def self.load_db
       SQLite3::Database.new DB_FILE
     end
@@ -39,14 +49,16 @@ module LogSense
     def self.locate_ip(ip, db)
       return unless ip
 
-      query = db.prepare 'SELECT * FROM ip_location where from_ip_n <= ? order by from_ip_n desc limit 1'
+      query = db.prepare "SELECT * FROM ip_location
+                            where from_ip_n <= ?
+                            order by from_ip_n desc limit 1"
       begin
         ip_n = IPAddr.new(ip).to_i
         result_set = query.execute ip_n
         country_code = result_set.map { |x| x[3] }[0]
         IsoCountryCodes.find(country_code).name
       rescue IPAddr::InvalidAddressError
-        'INVALID IP'
+        "INVALID IP"
       rescue IsoCountryCodes::UnknownCodeError
         country_code
       end

data/lib/log_sense/options_checker.rb

@@ -0,0 +1,24 @@
+module LogSense
+  #
+  # Check options and return appropriate error if
+  # command arguments are wrong
+  #
+  module OptionsChecker
+    SUPPORTED_CHAINS = {
+      rails: %i[txt html sqlite3 ufw],
+      apache: %i[txt html sqlite3 ufw]
+    }.freeze
+
+    def self.compatible?(iformat, oformat)
+      (SUPPORTED_CHAINS[iformat.to_sym] || []).include? oformat.to_sym
+    end
+
+    def self.chains_to_s
+      string = ""
+      SUPPORTED_CHAINS.each do |iformat, oformat|
+        string << "- #{iformat}: #{oformat.join(", ")}\n"
+      end
+      string
+    end
+  end
+end

data/lib/log_sense/options_parser.rb

@@ -1,6 +1,7 @@
-require 'optparse'
-require 'optparse/date'
-require 'log_sense/version'
+require "optparse"
+require "optparse/date"
+require "log_sense/version"
+require "log_sense/options_checker"
 
 module LogSense
   module OptionsParser
@@ -9,94 +10,121 @@ module LogSense
     #
     def self.parse(options)
       limit = 100
-      args = {} 
+      args = {}
 
       opt_parser = OptionParser.new do |opts|
-        opts.banner = 'Usage: log_sense [options] [logfile ...]'
+        opts.banner = "Usage: log_sense [options] [logfile ...]"
 
-        opts.on('-tTITLE', '--title=TITLE', String, 'Title to use in the report') do |n|
-          args[:title] = n
+        opts.on("-tTITLE", "--title=TITLE",
+                String,
+                "Title to use in the report") do |optval|
+          args[:title] = optval
         end
 
-        opts.on('-fFORMAT', '--input-format=FORMAT', String, 'Input format (either rails or apache)') do |n|
-          args[:input_format] = n
+        opts.on("-fFORMAT", "--input-format=FORMAT",
+                String,
+                "Input format (either rails or apache)") do |optval|
+          args[:input_format] = optval
         end
 
-        opts.on('-iFORMAT', '--input-files=file,file,', Array, 'Input files (can also be passed directly)') do |n|
-          args[:input_filenames] = n
+        opts.on("-iFORMAT", "--input-files=file,file,",
+                Array,
+                "Input files (can also be passed directly)") do |optval|
+          args[:input_filenames] = optval
         end
 
-        opts.on('-tFORMAT', '--output-format=FORMAT', String, 'Output format: html, org, txt, sqlite. See below for available formats') do |n|
-          args[:output_format] = n
+        opts.on("-tFORMAT", "--output-format=FORMAT",
+                String,
+                "Output format: html, org, txt, sqlite.") do |optval|
+          args[:output_format] = optval
         end
 
-        opts.on('-oOUTPUT_FILE', '--output-file=OUTPUT_FILE', String, 'Output file') do |n|
-          args[:output_file] = n
+        opts.on("-oOUTPUT_FILE", "--output-file=OUTPUT_FILE",
+                String,
+                "Output file") do |n|
+          args[:output_filename] = n
         end
 
-        opts.on('-bDATE', '--begin=DATE', Date, 'Consider entries after or on DATE') do |n|
-          args[:from_date] = n
+        opts.on("-bDATE", "--begin=DATE",
+                Date,
+                "Consider entries after or on DATE") do |optval|
+          args[:from_date] = optval
         end
 
-        opts.on('-eDATE', '--end=DATE', Date, 'Consider entries before or on DATE') do |n|
-          args[:to_date] = n
+        opts.on("-eDATE", "--end=DATE",
+                Date,
+                "Consider entries before or on DATE") do |optval|
+          args[:to_date] = optval
         end
 
-        opts.on('-lN', '--limit=N', Integer, "Limit to the N most requested resources (defaults to #{limit})") do |n|
-          args[:limit] = n
+        opts.on("-lN", "--limit=N",
+                Integer,
+                "Limit to the N most requested resources (defaults to #{limit})") do |optval|
+          args[:limit] = optval
         end
 
-        opts.on('-wWIDTH', '--width=WIDTH', Integer, 'Maximum width of long columns in textual reports') do |n|
-          args[:width] = n
+        opts.on("-wWIDTH", "--width=WIDTH",
+                Integer,
+                "Maximum width of long columns in textual reports") do |optval|
+          args[:width] = optval
         end
 
-        opts.on('-rROWS', '--rows=ROWS', Integer, 'Maximum number of rows for columns with multiple entries in textual reports') do |n|
-          args[:inner_rows] = n
+        opts.on("-rROWS", "--rows=ROWS",
+                Integer,
+                "Maximum number of rows for columns with multiple entries in textual reports") do |optval|
+          args[:inner_rows] = optval
         end
 
-        opts.on('-cPOLICY', '--crawlers=POLICY', String, 'Decide what to do with crawlers (applies to Apache Logs)') do |n|
-          case n
-          when 'only'
+        opts.on("-pPATTERN", "--pattern=PATTERN",
+                String,
+                "Pattern to use with ufw report to decide IP to blacklist") do |optval|
+          args[:pattern] = optval
+        end
+
+        opts.on("-cPOLICY", "--crawlers=POLICY",
+                String,
+                "Decide what to do with crawlers (applies to Apache Logs)") do |optval|
+          case optval
+          when "only"
             args[:only_crawlers] = true
-          when 'ignore'
+          when "ignore"
             args[:ignore_crawlers] = true
           end
         end
 
-        opts.on('-ns', '--no-selfpoll', 'Ignore self poll entries (requests from ::1; applies to Apache Logs)') do
+        opts.on("-ns", "--no-selfpoll",
+                "Ignore self poll entries (requests from ::1; applies to Apache Logs)") do
           args[:no_selfpoll] = true
         end
 
-        opts.on('--verbose', 'Inform about progress (prints to STDERR)') do
+        opts.on("-ng", "--no-geo",
+                "Do not geolocate entries") do
+          args[:geolocation] = false
+        end
+
+        opts.on("--verbose", "Inform about progress (output to STDERR)") do
           args[:verbose] = true
         end
 
-        opts.on('-v', '--version', 'Prints version information') do
+        opts.on("-v", "--version", "Prints version information") do
           puts "log_sense version #{LogSense::VERSION}"
-          puts 'Copyright (C) 2021 Shair.Tech'
-          puts 'Distributed under the terms of the MIT license'
+          puts "Copyright (C) 2021 Shair.Tech"
+          puts "Distributed under the terms of the MIT license"
           exit
         end
 
-        opts.on('-h', '--help', 'Prints this help') do
+        opts.on("-h", "--help", "Prints this help") do
           puts opts
-          puts ''
+          puts
           puts "This is version #{LogSense::VERSION}"
 
-          puts ''
-          puts 'Output formats'
-          pathname = File.join(File.dirname(__FILE__), 'templates', '*')
-          templates = Dir.glob(pathname).select { |x| !File.basename(x).start_with?(/_|#/) && !File.basename(x).end_with?('~') }
-          components = templates.map { |x| File.basename(x).split '.' }.group_by { |x| x[0] }
-          components.each do |k, vs|
-            puts "#{k} parsing can produce the following outputs:"
-            puts '  - sqlite'
-            vs.each do |v|
-              puts "  - #{v[1]}"
-            end
-          end
+          puts
+          puts "Output formats:"
+          puts
 
-          exit
+          puts OptionsChecker.chains_to_s
+
+          exit 0
         end
       end
 
@@ -104,12 +132,14 @@ module LogSense
 
       args[:limit] ||= limit
       args[:input_filenames] ||= []
-      args[:input_format] ||= 'apache'
-      args[:output_format] ||= 'html'
+      args[:input_format] ||= "apache"
+      args[:output_format] ||= "html"
       args[:ignore_crawlers] ||= false
       args[:only_crawlers] ||= false
       args[:no_selfpoll] ||= false
       args[:verbose] ||= false
+      # if set to false leave, otherwise set to true
+      args[:geolocation] = true unless args[:geolocation] == false
 
       args
     end

data/lib/log_sense/rails_aggregator.rb

@@ -0,0 +1,69 @@
+module LogSense
+  class RailsAggregator < Aggregator
+    def initialize(db, options = { limit: 900 })
+      @table = "Event"
+      @date_field = "started_at"
+      @url_field = "url"
+
+      @db = db
+      @options = options
+    end
+
+    #
+    # take a sqlite3 database and analyze data
+    #
+    # @ variables are automatically put in the returned data
+    #
+    def aggregate
+      aggregate_log_info
+      aggregate_statuses
+      aggregate_ips
+
+      @daily_distribution = @db.execute %(
+          SELECT date(started_at), #{human_readable_day}, count(started_at)
+                 from Event
+                 where #{filter}
+                 group by date(started_at)).gsub("\n", "")
+
+      @time_distribution = @db.execute %(
+          SELECT strftime("%H", started_at), count(started_at) from Event
+                 where #{filter}
+                 group by strftime('%H', started_at)).gsub("\n", "")
+
+      @performance = @db.execute %(
+          SELECT distinct(controller),
+                 count(controller),
+                 printf("%.2f", min(duration_total_ms)),
+                 printf("%.2f", avg(duration_total_ms)),
+                 printf("%.2f", max(duration_total_ms))
+                 from Event
+                 where #{filter}
+                 group by controller order by controller).gsub("\n", "")
+
+      @fatal = @db.execute %Q(
+          SELECT strftime("%Y-%m-%d %H:%M", started_at),
+                 ip,
+                 url,
+                 error.description,
+                 event.log_id
+                 FROM Event JOIN Error
+                 ON event.log_id == error.log_id
+                 WHERE #{filter} and exit_status == 'F').gsub("\n", "") || [[]]
+
+      @internal_server_error = @db.execute %Q(
+         SELECT strftime("%Y-%m-%d %H:%M", started_at), status, ip, url,
+                error.description,
+                event.log_id
+                FROM Event JOIN Error
+                ON event.log_id == error.log_id
+                WHERE #{filter} and substr(status, 1, 1) == '5').gsub("\n", "") || [[]]
+
+      @error = @db.execute %Q(
+         SELECT filename, log_id, context, description, count(log_id)
+                FROM Error
+                GROUP BY description).gsub("\n", "") || [[]]
+      
+      instance_vars_to_hash
+    end
+  end
+end

data/lib/log_sense/rails_log_parser.rb

@@ -1,68 +1,87 @@
-require 'sqlite3'
+require "sqlite3"
 
 module LogSense
-  module RailsLogParser
-    def self.parse(streams, options = {})
-      db = SQLite3::Database.new ':memory:'
-      db.execute 'CREATE TABLE IF NOT EXISTS Event(
-         id INTEGER PRIMARY KEY AUTOINCREMENT,
-         exit_status TEXT,
-         started_at TEXT,
-         ended_at TEXT,
-         log_id TEXT,
-         ip TEXT,
-         unique_visitor TEXT,
-         url TEXT,
-         controller TEXT,
-         html_verb TEXT,
-         status INTEGER,
-         duration_total_ms FLOAT,
-         duration_views_ms FLOAT,
-         duration_ar_ms FLOAT,
-         allocations INTEGER,
-         comment TEXT,
-         source_file TEXT,
-         line_number INTEGER
-      )'
+  #
+  # parse a Rails log file and return a SQLite3 DB
+  #
+  class RailsLogParser
+    #
+    # Tell users which format I can parse
+    #
+    def provide
+      [:rails]
+    end
 
-      ins = db.prepare("insert into Event(
-         exit_status,
-         started_at,
-         ended_at,
-         log_id,
-         ip,
-         unique_visitor,
-         url,
-         controller,
-         html_verb,
-         status,
-         duration_total_ms,
-         duration_views_ms,
-         duration_ar_ms,
-         allocations,
-         comment,
-         source_file,
-         line_number
-      )
-      values (#{Array.new(17, '?').join(', ')})")
+    def parse(streams, options = {})
+      db = SQLite3::Database.new ":memory:"
+      
+      db.execute <<-EOS
+        CREATE TABLE IF NOT EXISTS Event(
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          exit_status TEXT,
+          started_at TEXT,
+          ended_at TEXT,
+          log_id TEXT,
+          ip TEXT,
+          unique_visitor TEXT,
+          url TEXT,
+          controller TEXT,
+          html_verb TEXT,
+          status INTEGER,
+          duration_total_ms FLOAT,
+          duration_views_ms FLOAT,
+          duration_ar_ms FLOAT,
+          allocations INTEGER,
+          comment TEXT,
+          source_file TEXT,
+          line_number INTEGER
+         )
+      EOS
+
+      ins = db.prepare <<-EOS
+        insert into Event(
+          exit_status,
+          started_at,
+          ended_at,
+          log_id,
+          ip,
+          unique_visitor,
+          url,
+          controller,
+          html_verb,
+          status,
+          duration_total_ms,
+          duration_views_ms,
+          duration_ar_ms,
+          allocations,
+          comment,
+          source_file,
+          line_number
+         )
+         values (#{Array.new(17, "?").join(", ")})
+      EOS
 
-      db.execute 'CREATE TABLE IF NOT EXISTS Error(
+      db.execute <<-EOS
+        CREATE TABLE IF NOT EXISTS Error(
          id INTEGER PRIMARY KEY AUTOINCREMENT,
          log_id TEXT,
          context TEXT,
          description TEXT,
          filename TEXT,
          line_number INTEGER
-      )'
+        )
+      EOS
 
-      ins_error = db.prepare("insert into Error(
+      ins_error = db.prepare <<-EOS
+       insert into Error(
          log_id,
          context,
          description,
          filename,
          line_number
-      )
-      values (?, ?, ?, ?, ?)")
+       )
+       values (?, ?, ?, ?, ?)
+      EOS
 
       # requests in the log might be interleaved.
       #
@@ -93,7 +112,11 @@ module LogSense
 
           data = match_and_process_error line
           if data
-            ins_error.execute(data[:log_id], data[:context], data[:description], filename, line_number)
+            ins_error.execute(data[:log_id],
+                              data[:context],
+                              data[:description],
+                              filename,
+                              line_number)
             next
           end
           
@@ -199,7 +222,7 @@ module LogSense
     EXCEPTION = /[A-Za-z_0-9:]+(Error)?/
     ERROR_REGEXP = /^\[#{ID}\] (?<context>#{EXCEPTION}) \((?<description>(#{EXCEPTION})?.*)\):/
 
-    def self.match_and_process_error line
+    def match_and_process_error line
       matchdata = ERROR_REGEXP.match line
       if matchdata
         {
@@ -207,16 +230,13 @@ module LogSense
           context: matchdata[:context],
           description: matchdata[:description]
         }
-      else
-        nil
       end
     end
 
-
     # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
     STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Started #{VERB} "#{URL}" for #{IP} at/
 
-    def self.match_and_process_start line
+    def match_and_process_start line
       matchdata = STARTED_REGEXP.match line
       if matchdata
         {
@@ -226,8 +246,6 @@ module LogSense
           url: matchdata[:url],
           ip: matchdata[:ip]
         }
-      else
-        nil
       end
     end
 
@@ -237,7 +255,7 @@ module LogSense
     # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
     COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
 
-    def self.match_and_process_completed(line)
+    def match_and_process_completed(line)
       matchdata = (COMPLETED_REGEXP.match line)
       # exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
       if matchdata
@@ -252,23 +270,19 @@ module LogSense
           allocations: matchdata[:alloc],
           comment: ""
         }
-      else
-        nil
       end
     end
 
     # I, [2021-10-19T08:16:34.345162 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
     PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Processing by (?<controller>[^ ]+) as/
 
-    def self.match_and_process_processing_by line
+    def match_and_process_processing_by line
       matchdata = PROCESSING_REGEXP.match line
       if matchdata
         {
           log_id: matchdata[:id],
           controller: matchdata[:controller]
         }
-      else
-        nil
       end
     end
 
@@ -278,7 +292,7 @@ module LogSense
     # F, [2021-12-04T00:34:05.839269 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728] actionpack (5.2.4.4) lib/action_dispatch/middleware/debug_exceptions.rb:65:in `call'
     FATAL_REGEXP = /F, \[#{TIMESTAMP} #[0-9]+\] FATAL -- : \[#{ID}\] (?<comment>.*)$/
 
-    def self.match_and_process_fatal(line)
+    def match_and_process_fatal(line)
       matchdata = FATAL_REGEXP.match line
       if matchdata
         {
@@ -286,14 +300,14 @@ module LogSense
           log_id: matchdata[:id],
           comment: matchdata[:comment]
         }
-      else
-        nil
       end
     end
 
     # generate a unique visitor id from an event
-    def self.unique_visitor_id(event)
-      "#{DateTime.parse(event[:started_at] || event[:ended_at] || "1970-01-01").strftime("%Y-%m-%d")} #{event[:ip]}"
+    def unique_visitor_id(event)
+      date = event[:started_at] || event[:ended_at] || "1970-01-01"
+      "#{DateTime.parse(date).strftime("%Y-%m-%d")} #{event[:ip]}"
     end
   end
 end
+