apache_log_report 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 911c11b89b21140783332a66419e442ed366debad950a430af6265a5cc800129
+  data.tar.gz: 0f4d3bf245a2436412d7419d724c0c996d7ea0c6057a847f24cde376e6cf13c5
+SHA512:
+  metadata.gz: eed37e0f7c2098e2ca6192760ca5c9722ada8c0a5259d3c52fd9ae382b32a0d39f3da3e37bbcc9ab2446d92e35c3b538c1037ac48a4364f8d05ec2c58ab54fac
+  data.tar.gz: 563e8a68028d905bd8389ceefff7ed02cdb82715426fa1e2d5b47feeb471d75034e59b948a04899801e6d7b708f0dca7ff385111d163aef273b91f5c8e3bf2f4

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+*~

data/CHANGELOG.org

@@ -0,0 +1,27 @@
+#+TITLE: ChangeLog
+#+AUTHOR: Adolfo Villafiorita
+#+STARTUP: showall
+
+* Unreleased
+
+This changes are in the repository but not yet released to Rubygems.
+
+** New Functions and Changes
+
+** Fixes
+
+** Documentation
+
+** Code
+
+
+* Version 1.0.0
+
+** New Functions and Changes
+
+** Fixes
+
+** Documentation
+
+** Code
+

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in apache_log_report.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Adolfo Villafiorita
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.org

@@ -0,0 +1,30 @@
+#+TITLE: README
+#+AUTHOR: Adolfo Villafiorita
+#+STARTUP: showall
+
+* Introduction
+
+* Installation
+
+* Usage
+
+* Change Log
+
+See the [[file:CHANGELOG.org][CHANGELOG]] file.
+
+* Compatibility
+
+
+* Author and Contributors
+
+[[http://ict4g.net/adolfo][Adolfo Villafiorita]].
+
+* Known Bugs
+
+Some known bugs and an unknown number of unknown bugs.
+
+(See the open issues for the known bugs.)
+
+* License
+
+Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/apache_log_report.gemspec

@@ -0,0 +1,32 @@
+require_relative 'lib/apache_log_report/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "apache_log_report"
+  spec.version       = ApacheLogReport::VERSION
+  spec.authors       = ["Adolfo Villafiorita"]
+  spec.email         = ["adolfo.villafiorita@ict4g.net"]
+
+  spec.summary       = %q{Generate a request report in OrgMode format from an Apache log file.}
+  spec.description   = %q{Generate a request report in OrgMode format from an Apache log file.}
+  spec.homepage      = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org/"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
+  spec.metadata["changelog_uri"] = "https://www.ict4g.net/gitea/adolfo/apache_log_report/CHANGELOG.org"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "browser"
+  spec.add_dependency "sqlite3"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "apache_log_report"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/apache_log_report

@@ -0,0 +1,627 @@
+#!/usr/bin/env ruby
+
+require 'apache_log_report'
+require 'progressbar'
+
+LIMIT  = 30
+
+##############################################################################
+# MONKEY PATCH ARRAY AND HASH TO BUILD PIPES WHEN COMPUTING STATS
+
+class Array
+  # counts occurrences of each element in an array and returns a hash { element => count }
+  def uniq_with_count
+    h = Hash.new(0); self.each { |l| h[l] += 1 }; h
+  end
+end
+
+class Hash
+  # sort and limit entries of a hash. Returns an array which can be output
+  # sort by key or value, optionally reverse, optionally return a limited number of items
+  # {2014 => 10, 2015 => 20, 2010 => 30} => [[2014, 10], ...]
+  def prepare_for_output options = {}
+    sorted_a = []
+
+    if options[:sort] == :key
+      sorted_a = self.sort_by { |k, v| k }
+    elsif options[:sort]
+      sorted_a = self.sort_by { |k, v| v[options[:sort]] }
+    else
+      sorted_a = self.to_a
+    end
+
+    sorted_a = sorted_a.reverse if options[:reverse]
+    sorted_a = sorted_a[0, options[:limit]] if options[:limit]
+
+    sorted_a
+  end
+end
+
+##############################################################################
+# EMITTERS ARE USED TO OUTPUT TO DIFFERENT FORMATS
+
+class OrgTableEmitter
+  # print either of:
+  # [[ row_title, value, value, value, ...], ...]
+  # [[ row_title, {key: value, key: value, key: value}, ...], ...]
+  def self.emit table, options = {}
+    if table.size == 0
+      puts "No values to show"
+      return
+    end
+
+    puts "\n\n#{options[:title]}\n\n" if options[:title]
+
+    puts "#+NAME: #{options[:name]}" if options[:name]
+
+    # if table is in the form of a hash, transform it into an array
+    if table[0][1].class == Hash then
+      table = table.map { |x| [ x[0] ] + x[1].values }
+    end
+
+    # get longest row title and then longest column or use 50 and 10 as defaults
+    firstcol_length = options[:compact] ? 2 + table.map { |x| x[0].to_s.size }.max : 50
+    othercol_length = options[:compact] ? 2 + table.map { |x| x[1..-1].map { |x| x.to_s.size }.max }.max : 10
+    # take into account also the headers lengths'
+    headers_length = options[:headers] ? 2 + options[:headers][1..-1].map { |x| x.to_s.size }.max : 0
+    othercol_length = [othercol_length, headers_length].max
+
+    # build the formatting string
+    col_sizes = [ firstcol_length ] + [othercol_length] * table[0][1..-1].size
+    col_classes = table[0].map { |x| x.class.to_s }
+    col_formats = col_classes.each_with_index.map { |x, i| format_for(x, col_sizes[i]) }.join("") + "|"
+
+    # print header if asked to do so
+    if options[:headers]
+      puts (col_sizes.map { |x| "| %-#{x}s " }.join("") % options[:headers]) + "|"
+      puts (col_sizes.map { |x| "|#{"-" * (2 + x)}" }.join("")) + "|"
+    end
+
+    # print each table row
+    table.each do |row|
+      puts col_formats % row
+    end
+
+    puts "\n"
+  end
+
+  private 
+
+  def self.format_for klass, size
+    case klass
+    when "String"
+      "| %-#{size}s "
+    when "Integer"
+      "| %#{size}d "
+    when "Double"
+      "| %#{size}.2f "
+    when "Float"
+      "| %#{size}.2f "
+    else
+      "| %#{size}s "
+    end      
+  end
+
+end
+
+
+##############################################################################
+# PARSE OPTIONS
+
+
+##############################################################################
+# PARSE COMMAND LINE, SETS OPTIONS, PERFORM BASIC CHECKS, AND RUN
+
+def days log
+  (log.last[:date_time].to_date - log.first[:date_time].to_date).to_i
+end
+
+options = OptionParser.parse ARGV
+
+limit = options[:limit]
+from_date = options[:from_date]
+to_date = options[:to_date]
+
+ignore_crawlers = options[:ignore_crawlers]
+only_crawlers = options[:only_crawlers]
+distinguish_crawlers = options[:distinguish_crawlers]
+
+no_selfpoll = options[:no_selfpoll] 
+prefix = options[:prefix] ? "#{options[:prefix]}" : ""
+suffix = options[:suffix] ? "#{options[:suffix]}" : ""
+log_file = ARGV[0]
+
+
+if log_file and not File.exist? log_file
+  puts "Error: file #{log_file} does not exist"
+  exit 1
+end
+
+##############################################################################
+# COMPUTE THE STATISTICS
+
+started_at = Time.now
+log = LogParser.parse log_file, options
+days = days(log)
+
+log_no_selfpolls = log.select { |x| x[:ip] != "::1" }
+
+log_input = no_selfpoll ? log : log_no_selfpolls
+
+# get only requested entries
+log_filtered = log_input.select { |x|
+  (not from_date or from_date <= x[:date_time]) and
+    (not to_date or x[:date_time] <= to_date) and
+    (not ignore_crawlers or x[:bot] == false) and
+    (not only_crawlers or x[:bot] == true)
+}
+days_filtered = days(log_filtered)
+
+printf <<EOS
+#+TITLE: Apache Log Analysis: #{log_file}
+#+DATE: <#{Date.today}>
+#+STARTUP: showall
+#+OPTIONS: ^:{}
+#+HTML_HEAD: <link rel="stylesheet" type="text/css" href="ala-style.css" />
+#+OPTIONS: html-style:nil
+EOS
+
+puts "\n\n* Summary"
+
+OrgTableEmitter.emit [ ["Input file", log_file || "stdin"],
+                       ["Ignore crawlers", options[:ignore_crawlers] == true],
+                       ["Only crawlers", options[:only_crawlers] == true],
+                       ["Distinguish crawlers", options[:distinguish_crawlers] == true],
+                       ["No selfpoll", no_selfpoll],
+                       ["Filter by date", (options[:from_date] != nil or options[:to_date] != nil)],
+                       ["Prefix", prefix],
+                       ["Suffix", suffix]
+                     ],
+                     title: "** Log Analysis Request Summary",
+                     compact: true
+
+
+OrgTableEmitter.emit [ ["First request", log.first[:date_time]],
+                       ["Last request", log.last[:date_time]],
+                       ["Days", days.to_s]
+                     ],
+                     title: "** Logging Period",
+                     compact: true
+
+
+OrgTableEmitter.emit [ ["First day (filtered)", log_filtered.first[:date_time]],
+                       ["Last day (filtered)", log_filtered.last[:date_time]],
+                       ["Days (filtered)", days_filtered.to_s]
+                     ],
+                     title: "** Portion Analyzed",
+                     compact: true
+
+
+OrgTableEmitter.emit [ ["Log size", log.size],
+                       ["Self poll entries", log.size - log_no_selfpolls.size],
+                       ["Entries Parsed", log_input.size],
+                       ["Entries after filtering", log_filtered.size],
+                     ],
+                     title: "** Filtering",
+                     compact: true,
+                     name: "size"
+
+#
+# hits, unique visitors, and size per day
+# take an array of hashes, group by a lambda function, count hits, visitors, and tx data
+#
+def group_and_count log, key
+  matches = log.group_by { |x| key.call(x) }
+
+  h = {}
+
+  # each key in matches is an array of hashes (all log entries matching key)
+  matches.each do |k, v|
+    h[k] = {
+      hits: v.size,
+      visitors: v.uniq { |x| [ x[:date_time].to_date, x[:ip], x[:user_agent_string] ] }.count,
+      tx: v.map { |x| x[:size] }.inject(&:+) / 1024,
+    }
+  end
+
+  h
+end
+
+# like the previous function, but the count function is responsible of returning a hash with the desired data
+# the previous function is: group_and_generic_count log, key, lamnda { |v| { hits: v.size, visitors: v.uniq ..., tx: v.map ... } }
+def group_and_generic_count log, key, count
+  matches = log.group_by { |x| key.call(x) }
+
+  h = {}
+
+  # each key in matches is an array of hashes (all log entries matching key)
+  matches.each do |k, v|
+    h[k] = count.call(v)
+  end
+
+  h
+end
+
+
+def totals hash
+  h = Hash.new
+  [:hits, :visitors, :tx].each do |c|
+    h[c] = hash.values.map { |x| x[c] }.inject(&:+)
+  end
+  h
+end
+
+##############################################################################
+
+table = group_and_count log_filtered, lambda { |x| x[:date_time].to_date }
+totals = totals table
+
+OrgTableEmitter.emit [ ["Hits", totals[:hits]],
+                       ["Unique Visitors", totals[:visitors]],
+                       ["Hits / Unique Visitor", totals[:hits] / totals[:visitors].to_f],
+                       ["TX (Kb)", totals[:tx] ],
+                       ["TX (Kb) / Unique Visitor", totals[:tx] / totals[:visitors]],
+                     ],
+                     title: "* Totals",
+                     name: "totals",
+                     compact: true
+
+if (distinguish_crawlers)
+  bot_table = group_and_count log_filtered.select { |x| x[:bot] }, lambda { |x| x[:date_time].to_date }
+  bot_totals = totals bot_table
+
+  OrgTableEmitter.emit [ ["Hits", bot_totals[:hits]],
+                         ["Unique Visitors", bot_totals[:visitors]],
+                         ["Hits / Unique Visitor", bot_totals[:hits] / bot_totals[:visitors].to_f],
+                         ["TX (Kb)", bot_totals[:tx] ],
+                         ["TX (Kb) / Unique Visitor", bot_totals[:tx] / bot_totals[:visitors]],
+                       ],
+                       title: "** Bot Totals",
+                       name: "bot_totals",
+                       compact: true
+
+  vis_table = group_and_count log_filtered.select { |x| not x[:bot] }, lambda { |x| x[:date_time].to_date }
+  vis_totals = totals vis_table
+
+  OrgTableEmitter.emit [ ["Hits", vis_totals[:hits]],
+                         ["Unique Visitors", vis_totals[:visitors]],
+                         ["Hits / Unique Visitor", vis_totals[:hits] / vis_totals[:visitors].to_f],
+                         ["TX (Kb)", vis_totals[:tx] ],
+                         ["TX (Kb) / Unique Visitor", vis_totals[:tx] / vis_totals[:visitors]],
+                       ],
+                       title: "** Visitors Totals",
+                       name: "vis_totals",
+                       compact: true
+
+end
+
+enriched_table = Hash.new
+table.map { |k, v| enriched_table[k] = v.merge({ dow: k.wday, month: k.month }) }
+
+OrgTableEmitter.emit enriched_table.prepare_for_output(sort: :key),
+                     title: "* Daily Distribution",
+                     compact: true,
+                     headers: ["Day", "Hits", "Visitors", "Size", "Wday", "Month"],
+                     name: "daily_distribution"
+
+puts <<EOS
+#+BEGIN_SRC gnuplot :var data = daily_distribution :results output :exports both :file #{prefix}daily#{suffix}.svg
+reset 
+set grid ytics linestyle 0
+set grid xtics linestyle 0
+set terminal svg size 1200,800 fname 'Arial'
+
+set xdata time
+set timefmt "%Y-%m-%d"
+set format x "%a, %b %d"
+set xtics rotate by 60 right
+
+set title "Hits and Visitors"
+set xlabel "Date"
+set ylabel "Hits"
+set ylabel2 "Visits"
+
+set style fill transparent solid 0.2 noborder
+
+plot data using 1:2 with linespoints lw 3 lc rgb "#0000AA" pointtype 5 title "Hits" axes x1y2,     \\
+     data using 1:2 with filledcurves below x1 linecolor rgb "#0000AA" notitle axes x1y2, \\
+     data using 1:3 with linespoints lw 3 lc rgb "#AA0000" pointtype 7 title "Visitors", \\
+     data using 1:3 with filledcurves below x1 notitle linecolor rgb "#AA0000", \\
+     data using 1:($3+10):3 with labels notitle textcolor rgb "#AA0000", \\
+     data using 1:($2+100):2 with labels notitle textcolor rgb "#0000AA" axes x1y2
+#+END_SRC
+EOS
+
+#
+# distribution per hour
+#
+
+table = group_and_count log_filtered, lambda { |x| x[:date_time].hour }
+table_processed = table.prepare_for_output(sort: :key).map { |x|
+  ["%02d" % x[0] + ":00"] +
+    [ x[1].merge(hits_per_day: x[1][:hits] / days_filtered,
+                 visitors_per_day: x[1][:visitors] / days_filtered,
+                 tx_per_day: x[1][:tx] / days_filtered) ] }
+
+OrgTableEmitter.emit table_processed,
+                     title: "* Time Distribution",
+                     compact: true,
+                     headers: ["Time", "Hits", "Visitors", "Size (Kb)", "Hits/Day", "Visit/Day", "Size (Kb)/Day"],
+                     name: "time_distribution"
+
+puts <<EOS
+#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports both :file #{prefix}time#{suffix}.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+
+set title "Hits and Visitors"
+set xlabel "Date"
+set ylabel "Hits and Visits"
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#0000AA" title "Hits",     \\
+     data using 3 lc rgb "#AA0000" title "Visitors" axes x1y2, \\
+     data using ($0 - 0.2):($2 + 10):2 with labels title "" textcolor rgb("#0000AA"), \\
+     data using ($0 + 0.2):($3 + 10):3 with labels title "" textcolor rgb("#AA0000") axes x1y2
+#+END_SRC
+
+EOS
+
+puts <<EOS
+#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports both :file #{prefix}time-traffic#{suffix}.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+
+set title "Traffic"
+set xlabel "Date"
+set ylabel "Traffic"
+
+set style fill solid 0.50
+set boxwidth 0.6
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#00AA00" title "Traffic", \\
+     data using ($0):($2 + 10):2 with labels title "" textcolor rgb("#00AA00")
+#+END_SRC
+
+EOS
+
+#
+# most requested pages
+#
+
+log_success = log_filtered.select { |x| (x[:status][0] == "2" or x[:status][0] == "3") and x[:type] == ".html" }
+table = group_and_count log_success, lambda { |x| x[:uri] }
+
+OrgTableEmitter.emit table.prepare_for_output(sort: :hits, reverse: true, limit: limit),
+                     title: "* Most Requested Pages",
+                     compact: true,
+                     headers: ["Page", "Hits", "Visitors", "Size"],
+                     name: "pages"
+
+puts "Total of #{table.size} entries."
+
+#
+# most requested URIs
+#
+
+log_success = log_filtered.select { |x| (x[:status][0] == "2" or x[:status][0] == "3") and x[:type] != ".html" }
+table = group_and_count log_success, lambda { |x| x[:uri] }
+
+OrgTableEmitter.emit table.prepare_for_output(sort: :hits, reverse: true, limit: limit),
+                     title: "* Most Requested URIs",
+                     compact: true,
+                     headers: ["URI", "Hits", "Visitors", "Size"],
+                     name: "pages"
+
+puts "Total of #{table.size} entries."
+
+#
+# 404s (Pages)
+#
+
+table = log_filtered.select { |x| x[:status] == "404" and x[:type] == ".html" }.map { |x| x[:uri] }.uniq_with_count
+
+OrgTableEmitter.emit table.prepare_for_output(reverse: true, sort: 0, limit: limit),
+                     title: "* HTML 404s",
+                     compact: true,
+                     headers: ["Page", "Misses"],
+                     name: "page_miss"
+
+puts "Total of #{table.size} entries."
+
+#
+# 404s URIs
+#
+
+table = log_filtered.select { |x| x[:status] == "404" and x[:type] != ".html" }.map { |x| x[:uri] }.uniq_with_count
+
+OrgTableEmitter.emit table.prepare_for_output(reverse: true, sort: 0, limit: limit),
+                     title: "* HTML 404s",
+                     compact: true,
+                     headers: ["Page", "Misses"],
+                     name: "page_miss"
+
+puts "Total of #{table.size} entries."
+
+#
+# Attacks
+#
+def reasonable_response_type ext
+  [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].include? ext.downcase
+end
+
+table = log_filtered.select { |x| x[:status] != "200" and not reasonable_response_type(x[:type]) }.map { |x| x[:uri] }.uniq_with_count
+
+OrgTableEmitter.emit table.prepare_for_output(reverse: true, sort: 0, limit: limit),
+                     title: "* Possible Attacks",
+                     compact: true,
+                     headers: ["Request", "Count"],
+                     name: "attacks"
+
+puts "Total of #{table.size} entries."
+
+#
+# IPs
+#
+
+table = group_and_count log_success, lambda { |x| x[:ip] }
+
+OrgTableEmitter.emit table.prepare_for_output(sort: :key, reverse: true, limit: limit),
+                     title: "* IPs",
+                     compact: true,
+                     headers: ["IP", "Hits", "Visitors", "Size"],
+                     name: "ips"
+
+puts "Total of #{table.size} entries."
+
+#
+# Statuses, Browsers and Platforms
+#
+
+[:status, :browser, :platform].each do |what|
+
+  result = log_filtered.map { |x| x[what] }.uniq_with_count
+
+  OrgTableEmitter.emit result.prepare_for_output(sort: :key),
+                       title: "* #{what.to_s.capitalize}",
+                       compact: true,
+                       headers: [what.to_s.capitalize, "Hits"],
+                       name: what.to_s
+
+  puts <<EOS
+#+BEGIN_SRC gnuplot :var data = #{what.to_s} :results output :exports both :file #{prefix}#{what.to_s}#{suffix}.svg
+reset 
+set grid ytics linestyle 0
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
+     data using ($0):($2+100):2 with labels textcolor rgb "#0000AA"
+#+END_SRC
+EOS
+
+end
+
+#
+# Statuses by day
+#
+result = group_and_generic_count log_filtered,
+                                 lambda { |x| x[:date_time].to_date },
+                                 lambda { |x| h = Hash.new;
+                                              h["4xx"] = x.select { |y| y[:status][0] == "4" }.count;
+                                              h["3xx"] = x.select { |y| y[:status][0] == "3" }.count;
+                                              h["2xx"] = x.select { |y| y[:status][0] == "2" }.count;
+                                              h }
+
+OrgTableEmitter.emit result.prepare_for_output(sort: :key),
+                     title: "* Daily Status",
+                     compact: true,
+                     headers: ["Day", "4xx", "3xx", "2xx"],
+                     name: "daily_statuses"
+
+puts <<EOS
+#+BEGIN_SRC gnuplot :var data = daily_statuses :results output :exports both :file #{prefix}daily-statuses#{suffix}.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+
+set title "Daily Statuses"
+set xlabel "Date"
+set ylabel "Number of Hits"
+set xtics rotate by 60 right
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#CC0000" title "4xx",     \\
+     data using 3 lc rgb "#0000CC" title "3xx", \\
+     data using 4 lc rgb "#00AA00" title "2xx", \\
+     data using ($0 - 1. / 4):($2 + 0.5):2 with labels title "" textcolor rgb("#CC0000"), \\
+     data using ($0):($3 + 0.5):3 with labels title "" textcolor rgb("#0000CC"), \\
+     data using ($0 + 1. / 4):($4 + 0.5):4 with labels title "" textcolor rgb("#00AA00")
+#+END_SRC
+
+EOS
+
+#
+# Referer
+#
+result = group_and_count log_filtered, lambda { |x| begin
+                                                      URI(x[:referer]).host
+                                                    rescue Exception
+                                                      ""
+                                                    end  }
+good_result = result.reject! { |k| k == nil } 
+
+OrgTableEmitter.emit good_result.prepare_for_output(sort: :key),
+                     title: "* Referer",
+                     compact: true,
+                     headers: ["Referer", "Hits", "Visitors", "Size"],
+                     name: "referers"
+
+puts <<EOS
+#+BEGIN_SRC gnuplot :var data = referers :results output :exports both :file #{prefix}referers#{suffix}.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+set grid xtics linestyle 0
+
+set title "Referers"
+set xlabel "Date"
+set xtics rotate by 60 right
+set ylabel "Hits and Visits"
+
+set style fill solid 0.45
+set boxwidth 0.7
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#AA00AA" title "Hits",     \\
+     data using 3 lc rgb "#0AAAA0" title "Visits", \\
+     data using ($0 - 1. / 3):($2 + 50):2 with labels title "" textcolor rgb("#AA00AA"), \\
+     data using ($0 + 1. / 3):($3 + 50):3 with labels title "" textcolor rgb("#0AAAA0")
+#+END_SRC
+EOS
+
+puts <<EOS
+* Local Variables                                                  :noexport:
+# Local Variables:
+# org-confirm-babel-evaluate: nil
+# org-display-inline-images: t
+# end:
+EOS
+
+ended_at = Time.now
+duration = ended_at - started_at
+
+puts <<EOS
+** Performance
+
+| Analysis started at | #{started_at.to_s}           |
+| Analysis ended at   | #{ended_at.to_s}             |
+| Duration (sec)      | #{"%.3d" % duration }        |
+| Duration (min)      | #{"%.3d" % (duration / 60 )} |
+| Log size            | #{log.size}                  |
+| Entries Parsed      | #{log_input.size}            |
+| Lines/sec           | #{log_input.size / duration} |
+EOS

data/lib/apache_log_report.rb

@@ -0,0 +1,4 @@
+require 'apache_log_report/log_parser_sqlite3.rb'
+require 'apache_log_report/option_parser.rb'
+require 'apache_log_report/version.rb'
+

data/lib/apache_log_report/log_parser_hash.rb

@@ -0,0 +1,49 @@
+require 'date'
+require 'browser'
+
+class LogParserHash
+  # make a matchdata into a Hash.
+  # pure magic gotten from: http://zetcode.com/db/sqliteruby/connect/
+  # Used during parsing to simplify the generation of the hash.
+  class MatchData
+    def to_h
+      names.map(&:intern).zip(captures).to_h
+    end
+  end
+
+  def parse filename, options = {}
+    progressbar = ProgressBar.create(output: $stderr)  
+
+    content = filename ? File.readlines(filename) : ARGF.readlines
+    progressbar.total = content.size
+
+    # We parse combined log, which looks like:
+    #
+    # 66.249.70.16 - - [18/Aug/2020:23:03:00 +0200] "GET /eatc/assets/images/team/gunde.png HTTP/1.1" 200 61586 "-" "Googlebot-Image/1.0"
+    # 178.172.20.114 - - [25/Aug/2020:17:13:21 +0200] "GET /favicon.ico HTTP/1.1" 404 196 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0"
+    # we do not parse entries such as:
+    combined_regexp = /^(?<ip>\S+) \S+ (?<remote_log_name>\S+) \[(?<timestamp>[^\]]+)\] "(?<method>[A-Z]+) (?<uri>.+)? HTTP\/[0-9.]+" (?<status>[0-9]{3}) (?<size>[0-9]+|-) "(?<referer>[^"]*)" "(?<user_agent_string>[^"]+)"/
+
+    content.collect { |line|
+      hashie = combined_regexp.match line
+      hash = hashie.to_h
+
+      progressbar.increment
+
+      if hash != {}
+        hash[:date_time] = DateTime.parse(hash[:timestamp].sub(":", " "))
+        hash[:size] = hash[:size].to_i
+        hash[:type] = hash[:uri] ? File.extname(hash[:uri]) : ""
+
+        ua = Browser.new(hash[:user_agent_string], accept_language: "en-us")
+        hash[:bot] = ua.bot?
+        hash[:browser] = ua.name || ""
+        hash[:browser_version] = ua.version || ""
+        hash[:platform] = ua.platform.name || ""
+        hash[:platform_version] = ua.platform.version || ""
+
+        hash
+      end
+    }.compact
+  end
+end

data/lib/apache_log_report/log_parser_sqlite3.rb

@@ -0,0 +1,99 @@
+#
+# SQLITE3
+#
+require 'sqlite3'
+
+class LogParser
+  def self.parse filename, options = {}
+    
+    progressbar = ProgressBar.create(output: $stderr)  
+
+    content = filename ? File.readlines(filename) : ARGF.readlines
+    progressbar.total = content.size
+
+    db = SQLite3::Database.new ":memory:"
+    db.execute "CREATE TABLE IF NOT EXISTS LogLine(
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      date_time TEXT,
+      ip TEXT,
+      remote_log_name TEXT,
+      method TEXT,
+      uri TEXT,
+      status TEXT,
+      size INTEGER,
+      referer TEXT,
+      user_agent_string TEXT,
+      bot INTEGER,
+      browser TEXT,
+      browser_version TEXT,
+      platform TEXT,
+      platform_version TEXT
+       )"
+    
+    ins = db.prepare('insert into LogLine (
+                date_time, 
+                ip,
+                remote_log_name,
+                method,
+                uri, 
+                status,
+                size,
+                referer,
+                user_agent_string,
+                bot,
+                browser,
+                browser_version,
+                platform,
+                platform_version)
+              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+
+    combined_regexp = /^(?<ip>\S+) \S+ (?<remote_log_name>\S+) \[(?<datetime>[^\]]+)\] "(?<method>[A-Z]+) (?<uri>.+)? HTTP\/[0-9.]+" (?<status>[0-9]{3}) (?<size>[0-9]+|-) "(?<referer>[^"]*)" "(?<user_agent_string>[^"]+)"/
+
+    content.collect { |line|
+      hashie = combined_regexp.match line
+
+      progressbar.increment
+
+      puts hashie
+      if hashie != {}
+        ua = Browser.new(hashie[:user_agent_string], accept_language: "en-us")
+        puts <<EOS
+         #{hashie[:datetime].sub(":", " ")},
+         #{hashie[:ip]},
+         #{hashie[:remote_log_name]},
+         #{hashie[:method]},
+         #{hashie[:uri]},
+         #{ hashie[:status]},
+         #{ hashie[:size].to_i},
+         #{ hashie[:referer]},
+         #{ hashie[:user_agent_string]},
+         #{ ua.bot? ? 1 : 0},
+         #{ ua.name || ""},
+         #{ ua.version || ""},
+         #{ ua.platform.name || ""},
+         #{ ua.platform.version || ""}
+EOS
+
+        ins.execute(
+          hashie[:datetime].sub(":", " "),
+          hashie[:ip],
+          hashie[:remote_log_name],
+          hashie[:method],
+          hashie[:uri],
+          hashie[:status],
+          hashie[:size].to_i,
+          hashie[:referer],
+          hashie[:user_agent_string],
+          ua.bot? ? 1 : 0,
+          (ua.name || ""),
+          (ua.version || ""),
+          (ua.platform.name || ""),
+          (ua.platform.version || "")
+        )
+      end
+    }
+
+    db
+  end
+end
+

data/lib/apache_log_report/option_parser.rb

@@ -0,0 +1,63 @@
+require 'optparse'
+require 'optparse/date'
+
+class OptionParser
+  def self.parse(options)
+    args = {} 
+
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = "Usage: log-analyzer.rb [options] logfile"
+
+      opts.on("-lN", "--limit=N", Integer, "Number of entries to show (defaults to #{LIMIT})") do |n|
+        args[:limit] = n
+      end
+
+      opts.on("-bDATE", "--from-date=DATE", DateTime, "Consider entries after or on DATE") do |n|
+        args[:from_date] = n
+      end
+
+      opts.on("-eDATE", "--to-date=DATE", DateTime, "Consider entries before or on DATE") do |n|
+        args[:to_date] = n
+      end
+
+      opts.on("-i", "--ignore-crawlers", "Ignore crawlers") do |n|
+        args[:ignore_crawlers] = true
+      end
+
+      opts.on("-c", "--only-crawlers", "Perform analysis on crawlers only") do |n|
+        args[:only_crawlers] = true
+      end
+
+      opts.on("-t", "--distinguish-crawlers", "Print totals distinguishing crawlers from visitors") do |n|
+        args[:distinguish_crawlers] = true
+      end
+
+      opts.on("-p", "--ignore-selfpoll", "Ignore apaches self poll entries (from ::1)") do |n|
+        args[:no_selfpoll] = true
+      end
+
+      opts.on("-u", "--prefix=PREFIX", String, "Prefix to add to all plots (used to run multiple analyses in the same dir)") do |n|
+        args[:prefix] = n
+      end
+
+      opts.on("-w", "--suffix=SUFFIX", String, "Suffix to add to all plots (used to run multiple analyses in the same dir)") do |n|
+        args[:suffix] = n
+      end
+
+      opts.on("-h", "--help", "Prints this help") do
+        puts opts
+        exit
+      end
+    end
+
+    opt_parser.parse!(options)
+
+    args[:limit] ||= LIMIT
+    args[:ignore_crawlers] ||= false
+    args[:only_crawlers] ||= false
+    args[:distinguish_crawlers] ||= false
+    args[:no_selfpoll] ||= false
+
+    return args
+  end
+end

data/lib/apache_log_report/version.rb

@@ -0,0 +1,3 @@
+module ApacheLogReport
+  VERSION = "0.9.0"
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: apache_log_report
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Adolfo Villafiorita
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2020-09-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: browser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Generate a request report in OrgMode format from an Apache log file.
+email:
+- adolfo.villafiorita@ict4g.net
+executables:
+- apache_log_report
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG.org
+- Gemfile
+- LICENSE.txt
+- README.org
+- Rakefile
+- apache_log_report.gemspec
+- bin/console
+- bin/setup
+- exe/apache_log_report
+- lib/apache_log_report.rb
+- lib/apache_log_report/log_parser_hash.rb
+- lib/apache_log_report/log_parser_sqlite3.rb
+- lib/apache_log_report/option_parser.rb
+- lib/apache_log_report/version.rb
+homepage: https://www.ict4g.net/gitea/adolfo/apache_log_report
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org/
+  homepage_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
+  source_code_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
+  changelog_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report/CHANGELOG.org
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Generate a request report in OrgMode format from an Apache log file.
+test_files: []