log_sense 1.0.0

data/lib/log_sense/ip_locator.rb

@@ -0,0 +1,55 @@
+require 'csv'
+require 'sqlite3'
+require 'ipaddr'
+require 'iso_country_codes'
+
+module LogSense
+  module IpLocator
+    DB_FILE = "ip_locations/dbip-country-lite.sqlite3"
+
+    def self.dbip_to_sqlite db_location
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE ip_location (
+        from_ip_n INTEGER,
+        from_ip TEXT,
+        to_ip TEXT,
+        country_code TEXT
+      )"
+
+      ins = db.prepare "INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)"
+      CSV.foreach(db_location) do |row|
+        ip = IPAddr.new row[0]
+        ins.execute(ip.to_i, row[0], row[1], row[2])
+      end
+
+      # persist to file
+      ddb = SQLite3::Database.new(DB_FILE)
+      b = SQLite3::Backup.new(ddb, 'main', db, 'main')
+      b.step(-1) #=> DONE
+      b.finish
+    end
+
+    def self.load_db
+      SQLite3::Database.new DB_FILE
+    end
+
+    def self.locate_ip ip, db
+      ip_n = IPAddr.new(ip).to_i
+      res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
+      IsoCountryCodes.find(res[0][3]).name
+    end
+
+    #
+    # add country code to data[:ips]
+    #
+    def self.geolocate data
+      @location_db = IpLocator::load_db
+      data[:ips].each do |ip|
+        country_code = IpLocator::locate_ip ip[0], @location_db
+        ip << country_code
+      end
+      data
+    end
+
+  end
+end

data/lib/log_sense/options_parser.rb

@@ -0,0 +1,86 @@
+require 'optparse'
+require 'optparse/date'
+require 'apache_log_report/version'
+
+module LogSense
+  module OptionsParser
+    #
+    # parse command line options
+    #
+    def self.parse options
+      limit = 30
+      args = {} 
+
+      opt_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: log_sense [options] [logfile]"
+
+        opts.on("-fFORMAT", "--from=FORMAT", String, "Input format (either rails or apache)") do |n|
+          args[:input_format] = n
+        end
+
+        opts.on("-iINPUT_FILE", "--input=INPUT_FILE", String, "Input file") do |n|
+          args[:input_file] = n
+        end
+
+        opts.on("-tFORMAT", "--to=FORMAT", String, "Output format: html, org, txt, sqlite. Defaults to org mode") do |n|
+          args[:output_format] = n
+        end
+
+        opts.on("-oOUTPUT_FILE", "--output=OUTPUT_FILE", String, "Output file") do |n|
+          args[:output_file] = n
+        end
+
+        opts.on("-bDATE", "--begin=DATE", Date, "Consider entries after or on DATE") do |n|
+          args[:from_date] = n
+        end
+
+        opts.on("-eDATE", "--end=DATE", Date, "Consider entries before or on DATE") do |n|
+          args[:to_date] = n
+        end
+
+        opts.on("-lN", "--limit=N", Integer, "Number of entries to show (defaults to #{limit})") do |n|
+          args[:limit] = n
+        end
+
+        opts.on("-cPOLICY", "--crawlers=POLICY", String, "Decide what to do with crawlers (applies to Apache Logs)") do |n|
+          case n
+          when 'only'
+            args[:only_crawlers] = true
+          when 'ignore'
+            args[:ignore_crawlers] = true
+          end
+        end
+
+        opts.on("-np", "--ignore-selfpoll", "Ignore self poll entries (requests from ::1; applies to Apache Logs)") do
+          args[:no_selfpoll] = true
+        end
+
+        opts.on("-v", "--version", "Prints version information") do
+          puts "log_sense version #{LogSense::VERSION}"
+          puts "Copyright (C) 2020 Adolfo Villafiorita"
+          puts "Distributed under the terms of the MIT license"
+          puts ""
+          puts "Written by Adolfo Villafiorita"
+          exit
+        end
+
+        opts.on("-h", "--help", "Prints this help") do
+          puts opts
+          puts "This is version #{LogSense::VERSION}"
+          exit
+        end
+      end
+
+      opt_parser.parse!(options)
+
+      args[:limit] ||= limit
+      args[:input_format] ||= "apache"
+      args[:output_format] ||= "html"
+      args[:ignore_crawlers] ||= false
+      args[:only_crawlers] ||= false
+      args[:no_selfpoll] ||= false
+
+      return args
+    end
+  end
+end

data/lib/log_sense/rails_data_cruncher.rb

@@ -0,0 +1,117 @@
+module LogSense
+  module RailsDataCruncher
+
+    #
+    # take a sqlite3 database and analyze data
+    #
+    # @ variables are automatically put in the returned data
+    #
+
+    def self.crunch db, options = { limit: 30 }
+      first_day_s = db.execute "SELECT started_at from Event order by started_at limit 1"
+      # we could use ended_at to cover the full activity period, but I prefer started_at
+      # with the meaning that the monitor event initiation
+      last_day_s  = db.execute "SELECT started_at from Event order by started_at desc limit 1"
+
+      # make first and last day into dates or nil
+      # TODO: bug possible value here: [[nil]], which is not empty
+      @first_day = first_day_s.empty? ? nil : Date.parse(first_day_s[0][0])
+      @last_day = last_day_s.empty? ? nil : Date.parse(last_day_s[0][0])
+
+      @total_days = 0
+      if @first_day and @last_day
+        @total_days = (@last_day - @first_day).to_i
+      end
+
+      @events = db.execute "SELECT count(started_at) from Event"
+
+      @first_day_requested = options[:from_date]
+      @last_day_requested = options[:to_date]
+
+      @first_day_in_analysis = date_intersect options[:from_date], @first_day, :max
+      @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
+
+      @total_days_in_analysis = 0
+      if @first_day_in_analysis and @last_day_in_analysis
+        @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
+      end
+
+      #
+      # generate the where clause corresponding to the command line options to filter data
+      #
+      filter = [
+        (options[:from_date] ? "date(started_at) >= '#{options[:from_date]}'" : nil),
+        (options[:to_date] ? "date(started_at) <= '#{options[:to_date]}'" : nil),
+        "true"
+      ].compact.join " and "
+
+      mega = 1024 * 1024
+      giga = mega * 1024
+      tera = giga * 1024
+      
+      # in alternative to sum(size)
+      human_readable_size = <<-EOS
+      CASE 
+      WHEN sum(size) <  1024 THEN sum(size) || ' B' 
+      WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
+      WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
+      WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
+      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
+      END AS size
+      EOS
+
+      human_readable_day = <<-EOS
+        case cast (strftime('%w', started_at) as integer)
+          when 0 then 'Sunday'
+          when 1 then 'Monday'
+          when 2 then 'Tuesday'
+          when 3 then 'Wednesday'
+          when 4 then 'Thursday'
+          when 5 then 'Friday'
+          else 'Saturday'
+        end as dow
+      EOS
+
+      @total_events = db.execute "SELECT count(started_at) from Event where #{filter}"
+
+      @daily_distribution = db.execute "SELECT date(started_at), #{human_readable_day}, count(started_at) from Event where #{filter} group by date(started_at)"
+      @time_distribution = db.execute "SELECT strftime('%H', started_at), count(started_at) from Event where #{filter} group by strftime('%H', started_at)"
+
+      @statuses = db.execute "SELECT status, count(status) from Event where #{filter} group by status order by status"
+
+      @by_day_4xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '4' and #{filter} group by date(started_at)"
+      @by_day_3xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '3' and #{filter} group by date(started_at)"
+      @by_day_2xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '2' and #{filter} group by date(started_at)"
+
+      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
+        [x[0], x[1].map { |y| y[1] }].flatten
+      }
+
+      @ips = db.execute "SELECT ip, count(ip) from Event where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
+
+      @performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
+
+      data = {}
+      self.instance_variables.each do |variable|
+        var_as_symbol = variable.to_s[1..-1].to_sym
+        data[var_as_symbol] = eval(variable.to_s)
+      end
+      data
+    end
+
+    private
+
+    def self.date_intersect date1, date2, method
+      if date1 and date2
+        [date1, date2].send(method)
+      elsif date1
+        date1
+      else
+        date2
+      end
+    end
+
+
+  end
+end
+

data/lib/log_sense/rails_log_parser.rb

@@ -0,0 +1,176 @@
+require 'sqlite3'
+require 'byebug'
+
+module LogSense
+  module RailsLogParser
+    def self.parse filename, options = {}
+      content = filename ? File.readlines(filename) : ARGF.readlines
+
+      db = SQLite3::Database.new ":memory:"
+      db.execute 'CREATE TABLE IF NOT EXISTS Event(
+         id INTEGER PRIMARY KEY AUTOINCREMENT,
+         started_at TEXT,
+         ended_at TEXT,
+         log_id TEXT,
+         ip TEXT,
+         url TEXT,
+         controller TEXT,
+         html_verb TEXT,
+         status INTEGER, 
+         duration_total_ms FLOAT,
+         duration_views_ms FLOAT,
+         duration_ar_ms FLOAT,
+         allocations INTEGER
+      )'
+      
+      ins = db.prepare("insert into Event(
+         started_at,
+         ended_at,
+         log_id,
+         ip,
+         url,
+         controller,
+         html_verb,
+         status, 
+         duration_total_ms,
+         duration_views_ms,
+         duration_ar_ms,
+         allocations)
+      values (#{Array.new(12, '?').join(', ')})")
+
+      # requests in the log might be interleaved.
+      #
+      # We use the 'pending' variable to progressively store data
+      # about requests till they are completed; whey they are
+      # complete, we enter the entry in the DB and remove it from the
+      # hash
+      pending = {}
+
+      # Log lines are either one of:
+      #
+      # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Started VERB "URL" for IP at TIMESTAMP
+      # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Processing by CONTROLLER as FORMAT
+      # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Parameters: JSON
+      # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Rendered VIEW within LAYOUT (Duration: DURATION | Allocations: ALLOCATIONS)
+      # LOG_LEVEL, [ZULU_TIMESTAMP #NUMBER] INFO --: [ID] Completed STATUS STATUS_STRING in DURATION (Views: DURATION | ActiveRecord: DURATION | Allocations: NUMBER)
+      #
+      # and they appears in the order shown above: started, processing, ...
+      #
+      # Different requests might be interleaved, of course
+      
+      File.readlines(filename).each do |line|
+        # We discard LOG_LEVEL != 'I'
+        next if line[0] != 'I'
+        
+        data = self.match_and_process_start line
+        if data
+          id = data[:log_id]
+          pending[id] = data.merge (pending[id] || {})
+          next
+        end
+
+        data = self.match_and_process_processing_by line
+        if data
+          id = data[:log_id]
+          pending[id] = data.merge (pending[id] || {})
+          next
+        end
+
+        data = self.match_and_process_completed line
+        if data
+          id = data[:log_id]
+
+          # it might as well be that the first event started before
+          # the log.  With this, we make sure we add only events whose
+          # start was logged and parsed
+          if pending[id]
+            event = data.merge (pending[id] || {})
+
+            ins.execute(
+              event[:started_at],
+              event[:ended_at],
+              event[:log_id],
+              event[:ip],
+              event[:url],
+              event[:controller],
+              event[:html_verb],
+              event[:status],
+              event[:duration_total_ms],
+              event[:duration_views_ms],
+              event[:duration_ar_ms],
+              event[:allocations]
+            )
+
+            pending.delete(id)
+          end
+        end
+      end
+      
+      db
+    end
+
+    TIMESTAMP = /(?<timestamp>[^ ]+)/
+    ID = /(?<id>[a-z0-9-]+)/
+    VERB = /(?<verb>GET|POST|PATCH|PUT|DELETE)/
+    URL = /(?<url>[^"]+)/
+    IP = /(?<ip>[0-9.]+)/
+    STATUS = /(?<status>[0-9]+)/
+    MSECS = /[0-9.]+/
+
+    # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
+    STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Started #{VERB} "#{URL}" for #{IP} at/
+
+    def self.match_and_process_start line
+      matchdata = STARTED_REGEXP.match line
+      if matchdata
+        {
+          started_at: matchdata[:timestamp],
+          log_id: matchdata[:id],
+          html_verb: matchdata[:verb],
+          url: matchdata[:url],
+          ip: matchdata[:ip]
+        }
+      else
+        nil
+      end
+    end
+
+    # I, [2021-10-19T08:16:34.712331 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
+    COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms \| Allocations: (?<alloc>[0-9]+)\)/
+
+    def self.match_and_process_completed line
+      matchdata = COMPLETED_REGEXP.match line
+      if matchdata
+        {
+          ended_at: matchdata[:timestamp],
+          log_id: matchdata[:id],
+          status: matchdata[:status],
+          duration_total_ms: matchdata[:total],
+          duration_views_ms: matchdata[:views],
+          duration_ar_ms: matchdata[:arec],
+          allocations: matchdata[:alloc],
+        }
+      else
+        nil
+      end
+    end
+
+    # I, [2021-10-19T08:16:34.345162 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
+    PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Processing by (?<controller>[^ ]+) as/
+
+    def self.match_and_process_processing_by line
+      matchdata = PROCESSING_REGEXP.match line
+      if matchdata
+        {
+          log_id: matchdata[:id],
+          controller: matchdata[:controller]
+        }
+      else
+        nil
+      end
+    end
+
+  end
+
+end
+

data/lib/log_sense/templates/#apache.org.erb#

@@ -0,0 +1,266 @@
+#+TITLE: Apache Log Analysis: <%= data[:log_file] %>
+#+DATE: <<%= Date.today %>>
+#+STARTUP: showall
+#+OPTIONS: ^:{}
+#+HTML_HEAD: <link rel="stylesheet" type="text/css" href="ala-style.css" />
+#+OPTIONS: html-style:nil
+
+* Summary
+
+| Hits                | <%= "%10d" % data[:total_hits][0][0] %> |
+| Unique Visitors     | <%= "%10d" % data[:total_unique_visitors][0][0]  %> |
+| Tx                  | <%= "%10s" % data[:total_size][0][0] %> |
+| Logged Period       | <%= data[:first_day]  %> -- <%= data[:last_day] %> |
+| Days                | <%= "%10d" % data[:total_days]  %> |
+| Period Requested    | <%= data[:first_day_requested] %> -- <%= data[:last_day_requested] %> |
+| Period Analyzed     | <%= data[:first_day_in_analysis] %> -- <%= data[:last_day_in_analysis] %> |
+| Days in Analysis    | <%= data[:total_days_in_analysis] %> |
+
+* Daily Distribution
+
+<%= self.output_txt_table "daily_distribution", ["Day", "Hits", "Visits", "Size"], data[:daily_distribution]  %>
+
+#+BEGIN_SRC gnuplot :var data = daily_distribution :results output :exports <%= @export %> :file <%= @prefix %>daily<%= @suffix %>.svg
+reset 
+set grid ytics linestyle 0
+set grid xtics linestyle 0
+set terminal svg size 1200,800 fname 'Arial'
+
+set xdata time
+set timefmt "%Y-%m-%d"
+set format x "%a, %b %d"
+set xtics rotate by 60 right
+
+set title "Hits and Visitors"
+set xlabel "Date"
+set ylabel "Hits"
+set y2label "Visits"
+set y2tics
+
+set style fill transparent solid 0.2 noborder
+
+plot data using 1:2 with linespoints lw 3 lc rgb "#0000AA" pointtype 5 title "Hits" axes x1y2, \\
+data using 1:2 with filledcurves below x1 linecolor rgb "#0000AA" notitle axes x1y2, \\
+data using 1:3 with linespoints lw 3 lc rgb "#AA0000" pointtype 7 title "Visitors", \\
+data using 1:3 with filledcurves below x1 notitle linecolor rgb "#AA0000", \\
+data using 1:($3+0.1*$3):3 with labels notitle textcolor rgb "#AA0000", \\
+data using 1:($2+0.1*$2):2 with labels notitle textcolor rgb "#0000AA" axes x1y2
+#+END_SRC
+
+
+* Time Distribution
+
+<%=  self.output_txt_table "time_distribution", ["Hour", "Hits", "Visits", "Size"], data[:time_distribution]  %>
+
+
+#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time<%= @suffix %>.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+
+set title "Hits and Visitors"
+set xlabel "Date"
+set ylabel "Hits"
+set y2label "Visitors"
+set y2tics
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#0000AA" title "Hits",     \\
+data using 3 lc rgb "#AA0000" title "Visitors" axes x1y2, \\
+data using ($0 - 0.2):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#0000AA"), \\
+data using ($0 + 0.2):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#AA0000") axes x1y2
+#+END_SRC
+
+#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time-traffic<%= @suffix %>.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+
+set title "Traffic"
+set xlabel "Date"
+set ylabel "Traffic"
+
+set style fill solid 0.50
+set boxwidth 0.6
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#00AA00" title "Traffic", \\
+data using ($0):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00")
+#+END_SRC
+
+* Most Requested Pages
+
+<%=  self.output_txt_table "most_requested_pages", ["Path", "Hits", "Visits", "Size"], data[:most_requested_pages]  %>
+
+* Most Requested URIs
+
+<%=  self.output_txt_table "most_requested_resources", ["Path", "Hits", "Visits", "Size"], data[:most_requested_resources]  %>
+
+* 404s on HTML files
+
+<%=  self.output_txt_table "pages_404", ["Path", "Hits", "Visitors"], data[:missed_pages]  %>
+
+* 404s on other resources
+
+<%=  self.output_txt_table "resources_404", ["Path", "Hits", "Visitors"], data[:missed_resources]  %>
+
+* Possible Attacks
+
+<%=  self.output_txt_table "attacks", ["Path", "Hits", "Visitors"], data[:attacks]  %>
+
+* Statuses
+
+<%=  self.output_txt_table "statuses", ["Status", "Count"], data[:statuses]  %>
+
+#+BEGIN_SRC gnuplot :var data = statuses :results output :exports <%= @export %> :file <%= @prefix %>statuses<%= @suffix %>.svg
+reset 
+set grid ytics linestyle 0
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
+data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
+#+END_SRC
+
+* Daily Statuses
+
+<%=  self.output_txt_table "daily_statuses", ["Status", "2xx", "3xx", "4xx"], data[:statuses_by_day]  %>
+
+#+BEGIN_SRC gnuplot :var data = daily_statuses :results output :exports <%= @export %> :file <%= @prefix %>daily-statuses<%= @suffix %>.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+
+set title "Daily Statuses"
+set xlabel "Date"
+set ylabel "Number of Hits"
+set xtics rotate by 60 right
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#00AA00" title "2xx",     \\
+data using 3 lc rgb "#0000CC" title "3xx", \\
+data using 4 lc rgb "#AA0000" title "4xx", \\
+data using ($0 - 1. / 4):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00"), \\
+data using ($0):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0000CC"), \\
+data using ($0 + 1. / 4):($4 + 0.1*$4):4 with labels title "" textcolor rgb("#AA0000")
+#+END_SRC
+
+* Browsers
+
+<%=  self.output_txt_table "browsers", ["Browser", "Hits", "Visitors", "Size"], data[:browsers]  %>
+
+#+BEGIN_SRC gnuplot :var data = browsers :results output :exports <%= @export %> :file <%= @prefix %>browser<%= @suffix %>.svg
+reset 
+set grid ytics linestyle 0
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
+data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
+#+END_SRC
+
+* Platforms
+
+<%=  self.output_txt_table "platforms", ["Platform", "Hits", "Visitors", "Size"], data[:platforms]  %>
+
+#+BEGIN_SRC gnuplot :var data = platforms :results output :exports <%= @export %> :file <%= @prefix %>platforms<%= @suffix %>.svg
+reset 
+set grid ytics linestyle 0
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set style fill solid 0.25
+set boxwidth 0.6
+
+plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
+data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
+#+END_SRC
+
+* IPs
+
+<%=  self.output_txt_table "ips", ["IPs", "Hits", "Visitors", "Size"], data[:ips]  %>
+
+
+* Referers
+
+<%=  self.output_txt_tabl e"referers", ["Referers", "Hits", "Visitors", "Size"], data[:referers]  %>
+
+#+BEGIN_SRC gnuplot :var data = referers :results output :exports <%= @export %> :file <%= @prefix %>referers<%= @suffix %>.svg
+reset 
+set terminal svg size 1200,800 fname 'Arial' fsize 10
+
+set grid ytics linestyle 0
+set grid xtics linestyle 0
+
+set title "Referers"
+set xlabel "Date"
+set xtics rotate by 60 right
+set ylabel "Hits and Visits"
+
+set style fill solid 0.45
+set boxwidth 0.7
+
+set style data histograms
+set style histogram clustered gap 1
+
+plot data using 2:xtic(1) lc rgb "#AA00AA" title "Hits",     \\
+data using 3 lc rgb "#0AAAA0" title "Visits", \\
+data using ($0 - 1. / 3):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#AA00AA"), \\
+data using ($0 + 1. / 3):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0AAAA0")
+#+END_SRC
+
+* Command Invocation and Performance
+
+** Command Invocation
+
+#+BEGIN_EXAMPLE shell
+<%= data[:command] %>
+#+END_EXAMPLE
+
+| Input file           | <%= "%-50s" % (data[:log_file] || "stdin") %> |
+| Ignore crawlers      | <%= "%-50s" % options[:ignore_crawlers] %> |
+| Only crawlers        | <%= "%-50s" % options[:only_crawlers] %> |
+| No selfpoll          | <%= "%-50s" % options[:no_selfpoll] %> |
+| Filter by date       | <%= "%-50s" % (options[:from_date] != nil or options[:to_date] != nil) %> |
+| Prefix               | <%= "%-50s" % @prefix %> |
+| Suffix               | <%= "%-50s" % @suffix %> |
+
+** Log Structure
+
+| Log size                 | <%= "%10d" % data[:log_size][0][0] %> |
+| Self poll entries        | <%= "%10d" % data[:selfpolls_size][0][0] %> |
+| Crawlers                 | <%= "%10d" % data[:crawlers_size][0][0] %> |
+| Entries considered       | <%= "%10d" % data[:total_hits][0][0] %> |
+
+** Performance
+
+| Analysis started at | <%= data[:started_at].to_s %> |
+| Analysis ended at   | <%= data[:ended_at].to_s %> |
+| Duration (sec)      | <%= "%5.3d" % data[:duration]  %> |
+| Duration (min)      | <%= "%5.3d" % (data[:duration] / 60 ) %> |
+| Log size            | <%= "%9d"   % data[:log_size][0][0] %> |
+| Lines/sec           | <%= "%6.2f" % (data[:log_size][0][0] / data[:duration]) %> |
+
+* Local Variables                                                  :noexport:
+# Local Variables:
+# org-confirm-babel-evaluate: nil
+# org-display-inline-images: t
+# end: