log_sense 1.6.1 → 1.7.0

data/lib/log_sense/templates/_summary.html.erb

@@ -1,10 +1,10 @@
 <ul class="stats-list">
   <li>
-    <%= data[:first_day_in_analysis].strftime("%b %d, %Y") %>
+    <%= data[:first_day_in_analysis]&.strftime("%b %d, %Y") %>
     <span class="stats-list-label">From</span>
   </li>
   <li>
-    <%= data[:last_day_in_analysis].strftime("%b %d, %Y") %>
+    <%= data[:last_day_in_analysis]&.strftime("%b %d, %Y") %>
     <span class="stats-list-label">To</span>
   </li>
   <li class="stats-list-positive">

data/lib/log_sense/templates/report_html.erb

@@ -65,6 +65,7 @@
                      <%= report[:vega_spec].to_json %>,
                      { "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
                        width: "container",
+                       height: 400,
                        description: "<%= report[:title] %>",
                        data: {
                          values: data_<%= index %>

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "1.6.1"
+  VERSION = "1.7.0"
 end

data/lib/log_sense.rb

@@ -1,10 +1,10 @@
 require "log_sense/version"
 
-require "log_sense/options_parser"
-require "log_sense/options_checker"
+require "log_sense/options/parser"
+require "log_sense/options/checker"
 
-require "log_sense/apache_log_parser"
-require "log_sense/rails_log_parser"
+require "log_sense/apache/log_parser"
+require "log_sense/rails/log_parser"
 
 require "log_sense/aggregator"
 require "log_sense/apache_aggregator"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-04 00:00:00.000000000 Z
+date: 2024-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser
@@ -130,16 +130,16 @@ files:
 - ip_locations/dbip-country-lite.sqlite3
 - lib/log_sense.rb
 - lib/log_sense/aggregator.rb
+- lib/log_sense/apache/log_line_parser.rb
+- lib/log_sense/apache/log_parser.rb
 - lib/log_sense/apache_aggregator.rb
-- lib/log_sense/apache_log_line_parser.rb
-- lib/log_sense/apache_log_parser.rb
 - lib/log_sense/apache_report_shaper.rb
 - lib/log_sense/emitter.rb
 - lib/log_sense/ip_locator.rb
-- lib/log_sense/options_checker.rb
-- lib/log_sense/options_parser.rb
+- lib/log_sense/options/checker.rb
+- lib/log_sense/options/parser.rb
+- lib/log_sense/rails/log_parser.rb
 - lib/log_sense/rails_aggregator.rb
-- lib/log_sense/rails_log_parser.rb
 - lib/log_sense/rails_report_shaper.rb
 - lib/log_sense/report_shaper.rb
 - lib/log_sense/templates/_cdn_links.html.erb
@@ -188,7 +188,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.21
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Generate analytics for Apache and Rails log file.

data/lib/log_sense/apache_log_line_parser.rb

@@ -1,57 +0,0 @@
-module LogSense
-  # parses a log line and returns a hash
-  # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
-  #
-  # %h: IP
-  # %l: ident or -
-  # %u: userid or -
-  # %t: [10/Oct/2000:13:55:36 -0700]
-  #   day = 2*digit
-  #   month = 3*letter
-  #   year = 4*digit
-  #   hour = 2*digit
-  #   minute = 2*digit
-  #   second = 2*digit
-  #   zone = (`+' | `-') 4*digit
-  # %r: GET /apache_pb.gif HTTP/1.0
-  # %{User-agent}: "
-  #
-  # Example
-  # 116.179.32.16 - - [19/Dec/2021:22:35:11 +0100] "GET / HTTP/1.1" 200 135 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
-  #
-  class ApacheLogLineParser
-    DAY = /[0-9]{2}/
-    MONTH = /[A-Za-z]{3}/
-    YEAR = /[0-9]{4}/
-    TIMEC = /[0-9]{2}/
-    TIMEZONE = /(\+|-)[0-9]{4}/
-
-    IP = /(?<ip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|::1)/
-    IDENT = /(?<ident>[^ ]+|-)/
-    USERID = /(?<userid>[^ ]+|-)/
-
-    TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
-
-    HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
-    WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-    OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
-    METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-    PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
-    URL = /(?<url>[^ ]+)/
-    REFERER = /(?<referer>[^"]*)/
-    RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
-    SIZE = /(?<size>[0-9]+|-)/
-    USER_AGENT = /(?<user_agent>[^"]*)/
-
-    attr_reader :format
-
-    def initialize
-      @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
-    end
-
-    def parse(line)
-      @format.match(line) ||
-        raise("Apache LogLine Parser Error: Could not parse #{line}")
-    end
-  end
-end

data/lib/log_sense/apache_log_parser.rb

@@ -1,100 +0,0 @@
-require "sqlite3"
-require "browser"
-require "log_sense/apache_log_line_parser"
-
-module LogSense
-  #
-  # parse an Apache log file and return a SQLite3 DB
-  #
-  class ApacheLogParser
-    def parse(streams, options = {})
-      db = SQLite3::Database.new ":memory:"
-
-      db.execute "CREATE TABLE IF NOT EXISTS LogLine(
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      datetime TEXT,
-      ip TEXT,
-      user TEXT,
-      unique_visitor TEXT,
-      method TEXT,
-      path TEXT,
-      extension TEXT,
-      status TEXT,
-      size INTEGER,
-      referer TEXT,
-      user_agent TEXT,
-      bot INTEGER,
-      browser TEXT,
-      browser_version TEXT,
-      platform TEXT,
-      platform_version TEXT,
-      source_file TEXT,
-      line_number INTEGER
-      )"
-
-      ins = db.prepare("insert into LogLine (
-                datetime,
-                ip,
-                user,
-                unique_visitor,
-                method,
-                path,
-                extension,
-                status,
-                size,
-                referer,
-                user_agent,
-                bot,
-                browser,
-                browser_version,
-                platform,
-                platform_version,
-                source_file,
-                line_number
-                )
-              values (#{Array.new(18, '?').join(', ')})")
-
-      parser = ApacheLogLineParser.new
-
-      streams.each do |stream|
-        stream.readlines.each_with_index do |line, line_number|
-          begin
-            hash = parser.parse line
-            ua = Browser.new(hash[:user_agent], accept_language: 'en-us')
-            ins.execute(
-              DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
-              hash[:ip],
-              hash[:userid],
-              unique_visitor_id(hash),
-              hash[:method],
-              hash[:url],
-              (hash[:url] ? File.extname(hash[:url]) : ''),
-              hash[:status],
-              hash[:size].to_i,
-              hash[:referer],
-              hash[:user_agent],
-              ua.bot? ? 1 : 0,
-              (ua.name || ''),
-              (ua.version || ''),
-              (ua.platform.name || ''),
-              (ua.platform.version || ''),
-              stream == $stdin ? "stdin" : stream.path,
-              line_number
-            )
-          rescue StandardError => e
-            warn e.message
-          end
-        end
-      end
-
-      db
-    end
-
-    private
-
-    def unique_visitor_id hash
-      "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
-    end
-  end
-end
-

data/lib/log_sense/options_checker.rb

@@ -1,24 +0,0 @@
-module LogSense
-  #
-  # Check options and return appropriate error if
-  # command arguments are wrong
-  #
-  module OptionsChecker
-    SUPPORTED_CHAINS = {
-      rails: %i[txt html sqlite3 ufw],
-      apache: %i[txt html sqlite3 ufw]
-    }.freeze
-
-    def self.compatible?(iformat, oformat)
-      (SUPPORTED_CHAINS[iformat.to_sym] || []).include? oformat.to_sym
-    end
-
-    def self.chains_to_s
-      string = ""
-      SUPPORTED_CHAINS.each do |iformat, oformat|
-        string << "- #{iformat}: #{oformat.join(", ")}\n"
-      end
-      string
-    end
-  end
-end

data/lib/log_sense/options_parser.rb

@@ -1,147 +0,0 @@
-require "optparse"
-require "optparse/date"
-require "log_sense/version"
-require "log_sense/options_checker"
-
-module LogSense
-  module OptionsParser
-    #
-    # parse command line options
-    #
-    def self.parse(options)
-      limit = 100
-      args = {}
-
-      opt_parser = OptionParser.new do |opts|
-        opts.banner = "Usage: log_sense [options] [logfile ...]"
-
-        opts.on("-tTITLE", "--title=TITLE",
-                String,
-                "Title to use in the report") do |optval|
-          args[:title] = optval
-        end
-
-        opts.on("-fFORMAT", "--input-format=FORMAT",
-                String,
-                "Input format (either rails or apache)") do |optval|
-          args[:input_format] = optval
-        end
-
-        opts.on("-iFORMAT", "--input-files=file,file,",
-                Array,
-                "Input files (can also be passed directly)") do |optval|
-          args[:input_filenames] = optval
-        end
-
-        opts.on("-tFORMAT", "--output-format=FORMAT",
-                String,
-                "Output format: html, org, txt, sqlite.") do |optval|
-          args[:output_format] = optval
-        end
-
-        opts.on("-oOUTPUT_FILE", "--output-file=OUTPUT_FILE",
-                String,
-                "Output file") do |n|
-          args[:output_filename] = n
-        end
-
-        opts.on("-bDATE", "--begin=DATE",
-                Date,
-                "Consider entries after or on DATE") do |optval|
-          args[:from_date] = optval
-        end
-
-        opts.on("-eDATE", "--end=DATE",
-                Date,
-                "Consider entries before or on DATE") do |optval|
-          args[:to_date] = optval
-        end
-
-        opts.on("-lN", "--limit=N",
-                Integer,
-                "Limit to the N most requested resources (defaults to #{limit})") do |optval|
-          args[:limit] = optval
-        end
-
-        opts.on("-wWIDTH", "--width=WIDTH",
-                Integer,
-                "Maximum width of long columns in textual reports") do |optval|
-          args[:width] = optval
-        end
-
-        opts.on("-rROWS", "--rows=ROWS",
-                Integer,
-                "Maximum number of rows for columns with multiple entries in textual reports") do |optval|
-          args[:inner_rows] = optval
-        end
-
-        opts.on("-pPATTERN", "--pattern=PATTERN",
-                String,
-                "Pattern to use with ufw report to decide IP to blacklist") do |optval|
-          args[:pattern] = optval
-        end
-
-        opts.on("-cPOLICY", "--crawlers=POLICY",
-                String,
-                "Decide what to do with crawlers (applies to Apache Logs)") do |optval|
-          case optval
-          when "only"
-            args[:only_crawlers] = true
-          when "ignore"
-            args[:ignore_crawlers] = true
-          end
-        end
-
-        opts.on("-ns", "--no-selfpoll",
-                "Ignore self poll entries (requests from ::1; applies to Apache Logs)") do
-          args[:no_selfpoll] = true
-        end
-
-        opts.on("-ng", "--no-geo",
-                "Do not geolocate entries") do
-          args[:geolocation] = false
-        end
-
-        opts.on("--verbose", "Inform about progress (output to STDERR)") do
-          args[:verbose] = true
-        end
-
-        opts.on("-v", "--version", "Prints version information") do
-          puts "log_sense version #{LogSense::VERSION}"
-          puts "Copyright (C) 2021 Shair.Tech"
-          puts "Distributed under the terms of the MIT license"
-          exit
-        end
-
-        opts.on("-h", "--help", "Prints this help") do
-          puts opts
-          puts
-          puts "This is version #{LogSense::VERSION}"
-
-          puts
-          puts "Output formats:"
-          puts
-
-          puts OptionsChecker.chains_to_s
-
-          exit 0
-        end
-      end
-
-      opt_parser.parse!(options)
-
-      args[:limit] ||= limit
-      args[:input_filenames] ||= []
-      args[:input_format] ||= "apache"
-      args[:output_format] ||= "html"
-      args[:ignore_crawlers] ||= false
-      args[:only_crawlers] ||= false
-      args[:no_selfpoll] ||= false
-      args[:verbose] ||= false
-      # if set to false leave, otherwise set to true
-      args[:geolocation] = true unless args[:geolocation] == false
-
-      args
-    end
-  end
-end