apache_log_report 1.0.0 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be24b7fd62ff47195fe5e19cd2717252a37a687a8d92745add21acc952fc50a3
-  data.tar.gz: 3f5daff6a7447b6acca3a7ef456b148c3de49ff2dee87b75d419763dc2bbe2c4
+  metadata.gz: 45396921f6e594c2b2dcce25320c292c08f4bb35ab7c19dc72fb99cec8520dd4
+  data.tar.gz: 990dacd5e607a0f62a55f7248396ac095f2867f905807f91c3df1f6c06b5def0
 SHA512:
-  metadata.gz: d5271fd26059e942a730c649cd6ea00c35b459c3f5b130bd2d0ae707717b3554c90b7adff8364f384e56b5e8782ca147b1fd40f957290aed4a7bc443eb38387c
-  data.tar.gz: 475dfe3e49e76434c620bb75558bc2b5e40c6441fe455fca58d9b70b366a549f76a9d1bbbf9aae4bf5c9e0daa013770cefa9047fa11b8206f7eff4425be54b50
+  metadata.gz: a1c69d138504939d0c1917c7d9f278fff974fce0c27f82c64fd45b80641070924ea5f63a17b6efb4c64c63ea54f458a8b7d3d3f3e100664c70cdc3488be78950
+  data.tar.gz: a675921834c7551b2b95bf983c08a71f40f94f1f16e2b24d6b4fb10c4be61f9ffd8abb9947f985f0419d1678045758934e448d7e77f4904d806f3c966b8af358

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    apache_log_report (0.9.7)
+    apache_log_report (1.1.2)
       apache_log-parser
       browser
       sqlite3
@@ -11,12 +11,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     apache_log-parser (3.1.2)
-    browser (5.0.0)
+    browser (5.3.1)
     rake (12.3.3)
     sqlite3 (1.4.2)
-    terminal-table (1.8.0)
-      unicode-display_width (~> 1.1, >= 1.1.1)
-    unicode-display_width (1.7.0)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   ruby
@@ -26,4 +26,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.1.4
+   2.2.29

data/README.org

@@ -14,10 +14,8 @@ See the [[file:CHANGELOG.org][CHANGELOG]] file.
 
 * Todo
 
-** TODO Version information from command line and in reports
-** TODO Refactor code from one giant class to more manageable chunkes
-** TODO Move performance stats var to class (to isolate vars)
-** TODO Check total number of days (which is not working, now)
+** TODO Graphs in HTML output
+** TODO Countries
 
 * Compatibility
 

data/apache_log_report.gemspec

@@ -6,8 +6,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adolfo Villafiorita"]
   spec.email         = ["adolfo.villafiorita@ict4g.net"]
 
-  spec.summary       = %q{Generate a request report in OrgMode format from an Apache log file.}
-  spec.description   = %q{Generate a request report in OrgMode format from an Apache log file.}
+  spec.summary       = %q{Generate analytics from an Apache log file.}
+  spec.description   = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file.}
   spec.homepage      = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")

data/exe/apache_log_report

@@ -2,12 +2,13 @@
 
 require 'apache_log_report.rb'
 
+
 #
 # Parse Command Line Arguments
 #
 
 @command  = ARGV.join(" ")
-@options  = ApacheLogReport.options_parse ARGV
+@options  = ApacheLogReport::OptionsParser.parse ARGV
 @log_file = ARGV[0]
 
 if @log_file and not File.exist? @log_file
@@ -20,12 +21,30 @@ end
 #
 
 @started_at = Time.now
-@db = ApacheLogReport.parse @log_file
-ApacheLogReport.analyze_data @db, @options
-@ended_at = Time.now
-@duration = @ended_at - @started_at
 
-#
-# Emit Output
-#
-puts ApacheLogReport.emit @options, @command, @log_file, @started_at, @ended_at, @duration
+@db = ApacheLogReport::LogParser.parse @log_file
+
+if @options[:output_format]  == "sqlite"
+  ddb = SQLite3::Database.new('db.sqlite3')
+  b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
+  b.step(-1) #=> DONE
+  b.finish
+else
+  @data = ApacheLogReport::DataCruncher.crunch @db, @options
+
+  @ended_at = Time.now
+  @duration = @ended_at - @started_at
+
+  @data = @data.merge({
+                        command: @command,
+                        log_file: @log_file,
+                        started_at: @started_at,
+                        ended_at: @ended_at,
+                        duration: @duration
+                      })
+
+  #
+  # Emit Output
+  #
+  puts ApacheLogReport::Emitter.emit @data, @options
+end

data/lib/apache_log_report/apache_log_report.rb

@@ -0,0 +1,245 @@
+module ApacheLogReport
+
+  #
+  # parse command line options
+  #
+  require 'optparse'
+  require 'optparse/date'
+  require 'apache_log_report/version'
+
+  def self.options_parse options
+    limit = 30
+    args = {} 
+
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = "Usage: apache_log_report [options] [logfile]"
+
+      opts.on("-lN", "--limit=N", Integer, "Number of entries to show (defaults to #{limit})") do |n|
+        args[:limit] = n
+      end
+
+      opts.on("-bDATE", "--begin=DATE", DateTime, "Consider entries after or on DATE") do |n|
+        args[:from_date] = n
+      end
+
+      opts.on("-eDATE", "--end=DATE", DateTime, "Consider entries before or on DATE") do |n|
+        args[:to_date] = n
+      end
+
+      opts.on("-i", "--ignore-crawlers", "Ignore crawlers") do
+        args[:ignore_crawlers] = true
+      end
+
+      opts.on("-p", "--ignore-selfpoll", "Ignore apaches self poll entries (from ::1)") do
+        args[:no_selfpoll] = true
+      end
+
+      opts.on("-c", "--only-crawlers", "Perform analysis on crawlers only") do
+        args[:only_crawlers] = true
+      end
+
+      opts.on("-uPREFIX", "--prefix=PREFIX", String, "Prefix to add to all plots (used to run multiple analyses in the same dir)") do |n|
+        args[:prefix] = n
+      end
+
+      opts.on("-wSUFFIX", "--suffix=SUFFIX", String, "Suffix to add to all plots (used to run multiple analyses in the same dir)") do |n|
+        args[:suffix] = n
+      end
+
+      opts.on("-cWHAT", "--code-export=WHAT", String, "Control :export directive in code blocks (code, results, *both*, none)") do |n|
+        args[:code_export] = n
+      end
+      
+      opts.on("-v", "--version", "Prints version information") do
+        puts "apache_log_report version #{ApacheLogReport::VERSION}"
+        puts "Copyright (C) 2020 Adolfo Villafiorita"
+        puts "Distributed under the terms of the MIT license"
+        puts ""
+        puts "Written by Adolfo Villafiorita"
+        exit
+      end
+
+      opts.on("-h", "--help", "Prints this help") do
+        puts opts
+        puts "This is version #{ApacheLogReport::VERSION}"
+        exit
+      end
+    end
+
+    opt_parser.parse!(options)
+
+    args[:limit] ||= limit
+    args[:ignore_crawlers] ||= false
+    args[:no_selfpoll] ||= false
+    args[:only_crawlers] ||= false
+    args[:prefix] ||= ""
+    args[:suffix] ||= ""
+    args[:code_export] ||= "both"
+
+    return args
+  end
+
+  #
+  # parse an Apache log file and return a SQLite3 DB
+  #
+  require 'apache_log/parser'
+  require 'sqlite3'
+  require 'browser'
+
+  def self.parse filename, options = {}
+    content = filename ? File.readlines(filename) : ARGF.readlines
+
+    db = SQLite3::Database.new ":memory:"
+    db.execute "CREATE TABLE IF NOT EXISTS LogLine(
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      datetime TEXT,
+      ip TEXT,
+      user TEXT,
+      unique_visitor TEXT,
+      method TEXT,
+      path TEXT,
+      extension TEXT,
+      status TEXT,
+      size INTEGER,
+      referer TEXT,
+      user_agent TEXT,
+      bot INTEGER,
+      browser TEXT,
+      browser_version TEXT,
+      platform TEXT,
+      platform_version TEXT)"
+    
+    ins = db.prepare('insert into LogLine (
+                datetime, 
+                ip,
+                user,
+                unique_visitor,
+                method,
+                path, 
+                extension,
+                status,
+                size,
+                referer,
+                user_agent,
+                bot,
+                browser,
+                browser_version,
+                platform,
+                platform_version)
+              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+
+    parser = ApacheLog::Parser.new(options[:format] || 'combined')
+    
+    content.each do |line|
+      begin
+        hash = parser.parse line
+
+        ua = Browser.new(hash[:user_agent], accept_language: "en-us")
+        ins.execute(
+          hash[:datetime].iso8601,
+          hash[:remote_host],
+          hash[:user],
+          hash[:datetime].strftime("%Y-%m-%d") + " " + hash[:remote_host] + " " + hash[:user_agent],
+          hash[:request][:method],
+          hash[:request][:path],
+          (hash[:request][:path] ? File.extname(hash[:request][:path]) : ""),
+          hash[:status],
+          hash[:size].to_i,
+          hash[:referer],
+          hash[:user_agent],
+          ua.bot? ? 1 : 0,
+          (ua.name || ""),
+          (ua.version || ""),
+          (ua.platform.name || ""),
+          (ua.platform.version || "")
+        )
+      rescue
+        STDERR.puts "Apache Log parser error: could not parse #{line}"
+      end
+    end
+    
+    db
+  end
+
+  #
+  # take a sqlite3 database and analyze data
+  #
+  def self.analyze_data db, options = {}
+
+    @first_day      = db.execute "SELECT datetime from LogLine order by datetime limit 1"
+    @last_day       = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
+    @log_size       = db.execute "SELECT count(datetime) from LogLine"
+    @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+    @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
+
+    #
+    # generate the where clause corresponding to the command line options to filter data
+    #
+    @filter = [
+      (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
+      (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
+      (options[:only_crawlers] ? "bot == 1" : nil),
+      (options[:ignore_crawlers] ? "bot == 0" : nil),
+      (options[:no_selfpolls] ? "ip != '::1'" : nil),
+      "true"
+    ].compact.join " and "
+
+    # in alternative to sum(size)
+    human_readable_size = <<-EOS
+    CASE 
+    WHEN sum(size) < 1024 THEN sum(size) || ' B' 
+    WHEN sum(size) >=  1024 AND sum(size) < (1024 * 1024) THEN ROUND((CAST(sum(size) AS REAL) / 1024),2) || ' KB' 
+    WHEN sum(size) >= (1024 * 1024)  AND sum(size) < (1024 * 1024 * 1024) THEN ROUND((CAST(sum(size) AS REAL) / (1024 * 1024)),2) || ' MB' 
+    WHEN sum(size) >= (1024 * 1024 * 1024) AND sum(size) < (1024 * 1024 * 1024 * 1024) THEN ROUND((CAST(sum(size) AS REAL) / (1024 * 1024 * 1024)),2) || ' GB' 
+    WHEN sum(size) >= (1024 * 1024 * 1024 * 1024) THEN ROUND((CAST(sum(size) AS REAL) / (1024 * 1024 * 1024 * 1024)),2) || ' TB' 
+    END AS size
+    EOS
+
+    @total_hits = db.execute "SELECT count(datetime) from LogLine where #{@filter}"
+    @total_unique_visitors = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{@filter}"
+    @total_size = db.execute "SELECT #{human_readable_size} from LogLine where #{@filter}"
+    @total_days = (Date.parse(@last_day[0][0]) - Date.parse(@first_day[0][0])).to_i
+
+    @daily_distribution = db.execute "SELECT date(datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{@filter} group by date(datetime)"
+
+    @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{@filter} group by strftime('%H', datetime)"
+
+    @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where extension == '.html' and #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+    @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+    @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and extension == '.html' and #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+    @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+    @reasonable_requests_exts = [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].map {  |x|
+      "extension != '#{x}'"
+    }.join " and "
+
+    @attacks = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{@filter} and (#{@reasonable_requests_exts}) group by path order by count(path) desc  limit #{options[:limit]}"
+
+    @statuses = db.execute "SELECT status, count(status) from LogLine where #{@filter} group by status order by status"
+
+    @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{@filter} group by date(datetime)"
+    @by_day_3xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '3' and #{@filter} group by date(datetime)"
+    @by_day_2xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '2' and #{@filter} group by date(datetime)"
+
+    @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
+      [x[0], x[1].map { |y| y[1] }].flatten
+    }
+
+    @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by browser order by count(browser) desc"
+
+    @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by platform order by count(platform) desc"
+
+    @ips =  db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by ip order by count(ip) desc limit #{options[:limit]}"
+
+    @referers =  db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by referer order by count(referer) desc limit #{options[:limit]}"
+
+
+  end
+
+
+end
+
+

data/lib/apache_log_report/data_cruncher.rb

@@ -0,0 +1,86 @@
+module ApacheLogReport
+  module DataCruncher
+
+    #
+    # take a sqlite3 database and analyze data
+    #
+
+    def self.crunch db, options = {}
+      @first_day      = db.execute "SELECT datetime from LogLine order by datetime limit 1"
+      @last_day       = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
+      @log_size       = db.execute "SELECT count(datetime) from LogLine"
+      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
+
+      #
+      # generate the where clause corresponding to the command line options to filter data
+      #
+      filter = [
+        (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
+        (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
+        (options[:only_crawlers] ? "bot == 1" : nil),
+        (options[:ignore_crawlers] ? "bot == 0" : nil),
+        (options[:no_selfpolls] ? "ip != '::1'" : nil),
+        "true"
+      ].compact.join " and "
+
+      mega = 1024 * 1024
+      giga = mega * 1024
+      tera = giga * 1024
+      
+      # in alternative to sum(size)
+      human_readable_size = <<-EOS
+      CASE 
+      WHEN sum(size) <  1024 THEN sum(size) || ' B' 
+      WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
+      WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
+      WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
+      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
+      END AS size
+      EOS
+
+      @total_hits = db.execute "SELECT count(datetime) from LogLine where #{filter}"
+      @total_unique_visitors = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{filter}"
+      @total_size = db.execute "SELECT #{human_readable_size} from LogLine where #{filter}"
+      @total_days = (Date.parse(@last_day[0][0]) - Date.parse(@first_day[0][0])).to_i
+
+      @daily_distribution = db.execute "SELECT date(datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
+      @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"
+      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+      @reasonable_requests_exts = [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].map {  |x|
+        "extension != '#{x}'"
+      }.join " and "
+
+      @attacks = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} and (#{@reasonable_requests_exts}) group by path order by count(path) desc  limit #{options[:limit]}"
+      @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
+
+      @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"
+      @by_day_3xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '3' and #{filter} group by date(datetime)"
+      @by_day_2xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '2' and #{filter} group by date(datetime)"
+
+      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
+        [x[0], x[1].map { |y| y[1] }].flatten
+      }
+
+      @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
+      @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
+      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
+      @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
+      
+      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+
+      data = {}
+      self.instance_variables.each do |variable|
+        var_as_symbol = variable.to_s[1..-1].to_sym
+        data[var_as_symbol] = eval(variable.to_s)
+      end
+      data
+    end
+  end
+
+end
+

data/lib/apache_log_report/emitter.rb

@@ -0,0 +1,49 @@
+require 'terminal-table'
+require 'erb'
+require 'ostruct'
+
+module ApacheLogReport
+  module Emitter
+
+    #
+    # Emit Data
+    #
+    def self.emit data = {}, options = {}
+      @prefix = options[:prefix]
+      @suffix = options[:suffix]
+      @export = options[:code_export]
+      @mode = options[:output_format] || "org"
+      
+      # for the ERB binding
+      @data = data
+      @options = options
+
+      # determine the main template to read
+      @template = File.join(File.dirname(__FILE__), "templates", "template.#{@mode}.erb")
+      erb_template = File.read @template
+      
+      output = ERB.new(erb_template).result(binding)
+      puts output
+    end
+
+    private
+
+    def self.output_table name, headings, rows
+      name = "#+NAME: #{name}"
+      table = Terminal::Table.new headings: headings, rows: rows, style: { border_x: "-", border_i: "|" }
+
+      #(2..headings.size).each do |i| 
+      #  table.align_column(i, :right)
+      #end
+
+      name + "\n" + table.to_s
+    end
+
+    def self.render(template, vars)
+      @template = File.join(File.dirname(__FILE__), "templates", "_#{template}.html.erb")
+      erb_template = File.read @template
+      ERB.new(erb_template).result(OpenStruct.new(vars).instance_eval { binding })
+    end
+
+  end
+end

data/lib/apache_log_report/log_parser.rb

@@ -0,0 +1,87 @@
+require 'apache_log/parser'
+require 'sqlite3'
+require 'browser'
+
+module ApacheLogReport
+  module LogParser
+    #
+    # parse an Apache log file and return a SQLite3 DB
+    #
+
+    def self.parse filename, options = {}
+      content = filename ? File.readlines(filename) : ARGF.readlines
+
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE IF NOT EXISTS LogLine(
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      datetime TEXT,
+      ip TEXT,
+      user TEXT,
+      unique_visitor TEXT,
+      method TEXT,
+      path TEXT,
+      extension TEXT,
+      status TEXT,
+      size INTEGER,
+      referer TEXT,
+      user_agent TEXT,
+      bot INTEGER,
+      browser TEXT,
+      browser_version TEXT,
+      platform TEXT,
+      platform_version TEXT)"
+      
+      ins = db.prepare('insert into LogLine (
+                datetime, 
+                ip,
+                user,
+                unique_visitor,
+                method,
+                path, 
+                extension,
+                status,
+                size,
+                referer,
+                user_agent,
+                bot,
+                browser,
+                browser_version,
+                platform,
+                platform_version)
+              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+
+      parser = ApacheLog::Parser.new(options[:format] || 'combined')
+      
+      content.each do |line|
+        begin
+          hash = parser.parse line
+
+          ua = Browser.new(hash[:user_agent], accept_language: "en-us")
+          ins.execute(
+            hash[:datetime].iso8601,
+            hash[:remote_host],
+            hash[:user],
+            hash[:datetime].strftime("%Y-%m-%d") + " " + hash[:remote_host] + " " + hash[:user_agent],
+            hash[:request][:method],
+            hash[:request][:path],
+            (hash[:request][:path] ? File.extname(hash[:request][:path]) : ""),
+            hash[:status],
+            hash[:size].to_i,
+            hash[:referer],
+            hash[:user_agent],
+            ua.bot? ? 1 : 0,
+            (ua.name || ""),
+            (ua.version || ""),
+            (ua.platform.name || ""),
+            (ua.platform.version || "")
+          )
+        rescue
+          STDERR.puts "Apache Log parser error: could not parse #{line}"
+        end
+      end
+      
+      db
+    end
+
+  end
+end

data/lib/apache_log_report/options_parser.rb

@@ -0,0 +1,86 @@
+require 'optparse'
+require 'optparse/date'
+require 'apache_log_report/version'
+
+module ApacheLogReport
+  module OptionsParser
+    #
+    # parse command line options
+    #
+    def self.parse options
+      limit = 30
+      args = {} 
+
+      opt_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: apache_log_report [options] [logfile]"
+
+        opts.on("-lN", "--limit=N", Integer, "Number of entries to show (defaults to #{limit})") do |n|
+          args[:limit] = n
+        end
+
+        opts.on("-bDATE", "--begin=DATE", DateTime, "Consider entries after or on DATE") do |n|
+          args[:from_date] = n
+        end
+
+        opts.on("-eDATE", "--end=DATE", DateTime, "Consider entries before or on DATE") do |n|
+          args[:to_date] = n
+        end
+
+        opts.on("-i", "--ignore-crawlers", "Ignore crawlers") do
+          args[:ignore_crawlers] = true
+        end
+
+        opts.on("-p", "--ignore-selfpoll", "Ignore apaches self poll entries (from ::1)") do
+          args[:no_selfpoll] = true
+        end
+
+        opts.on("-c", "--only-crawlers", "Perform analysis on crawlers only") do
+          args[:only_crawlers] = true
+        end
+
+        opts.on("-uPREFIX", "--prefix=PREFIX", String, "Prefix to add to all plots (used to run multiple analyses in the same dir)") do |n|
+          args[:prefix] = n
+        end
+
+        opts.on("-wSUFFIX", "--suffix=SUFFIX", String, "Suffix to add to all plots (used to run multiple analyses in the same dir)") do |n|
+          args[:suffix] = n
+        end
+
+        opts.on("-cWHAT", "--code-export=WHAT", String, "Control :export directive in Org Mode code blocks (code, results, *both*, none)") do |n|
+          args[:code_export] = n
+        end
+        
+        opts.on("-fFORMAT", "--format=FORMAT", String, "Output format: html, org, sqlite. Defaults to org mode") do |n|
+          args[:output_format] = n
+        end
+
+        opts.on("-v", "--version", "Prints version information") do
+          puts "apache_log_report version #{ApacheLogReport::VERSION}"
+          puts "Copyright (C) 2020 Adolfo Villafiorita"
+          puts "Distributed under the terms of the MIT license"
+          puts ""
+          puts "Written by Adolfo Villafiorita"
+          exit
+        end
+
+        opts.on("-h", "--help", "Prints this help") do
+          puts opts
+          puts "This is version #{ApacheLogReport::VERSION}"
+          exit
+        end
+      end
+
+      opt_parser.parse!(options)
+
+      args[:limit] ||= limit
+      args[:ignore_crawlers] ||= false
+      args[:no_selfpoll] ||= false
+      args[:only_crawlers] ||= false
+      args[:prefix] ||= ""
+      args[:suffix] ||= ""
+      args[:code_export] ||= "both"
+
+      return args
+    end
+  end
+end