log_sense 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 646e45e06b2dd991574f0b7976d0e79e5e59c4230fb704650545aab1ad9304cd
-  data.tar.gz: 95251d0aeade9cdcbebd382df2d816f12f139b07f658d8523591bdcc6dde51e8
+  metadata.gz: 8f2fcc8402c27ec959854e3cd032e9e75693e538d3ca0879de3f4cf0b6aae9c5
+  data.tar.gz: e2530ff646c436ca88e674a698bf71876f2e62bc0b59f20ebbc4e38fdd787830
 SHA512:
-  metadata.gz: 0f2abde64d922b9a871a8a17d225672a5a1148165e9464c0cdde73cd456602c4221b08d5e572a015ffbc8b6b1c28a8c27a0f043b875c9590527930bed778f0c2
-  data.tar.gz: e27d0edd01a837b53bdcf0400499d52e8ffcb73796ff60fe11d7893d476521d519d8feb1614e6b98ac9af44faeb5a556a19afaf8e8f83c6cf634ec5055ef3fea
+  metadata.gz: 30932a09a0a0d0717c4e5caa9a579bb60a3a930dc144dc318029945e0364e71e5ddf5d5868ccf758a11d3ed5f73f5d690f482521e3475f1f16a5a041fa4abd8a
+  data.tar.gz: da8ac76f281cf9f8c67805621fe098d1480889f2fb5a55a378d1ffae3383c1913d572c28c9a68c876fb710b4e8bc3213b157c51cdf008f1505c320ef5fabedd0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.2.0)
+    log_sense (1.0.7)
       apache_log-parser
       browser
       ipaddr
@@ -34,4 +34,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.2.29
+   2.2.32

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2020 Adolfo Villafiorita
+Copyright (c) 2021 Shair.Tech
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/exe/log_sense

@@ -6,10 +6,11 @@ require 'log_sense.rb'
 # Parse Command Line Arguments
 #
 
-# better be here... OptionsParser consumes ARGV
+# this better be here... OptionsParser consumes ARGV
 @command_line = ARGV.join(" ")
+
 @options     = LogSense::OptionsParser.parse ARGV
-@input_file  = @options[:input_file]
+@input_file  = @options[:input_file] || ARGV[0]
 @output_file = @options[:output_file]
 
 if not @input_file

data/lib/log_sense/apache_data_cruncher.rb

@@ -11,8 +11,8 @@ module LogSense
       last_day_s  = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
 
       # make first and last day into dates or nil
-      @first_day = first_day_s.empty? ? nil : Date.parse(first_day_s[0][0])
-      @last_day = last_day_s.empty? ? nil : Date.parse(last_day_s[0][0])
+      @first_day = first_day_s[0][0] ? nil : Date.parse(first_day_s[0][0])
+      @last_day = last_day_s[0][0] ? nil : Date.parse(last_day_s[0][0])
 
       @total_days = 0
       if @first_day and @last_day
@@ -20,8 +20,13 @@ module LogSense
       end
 
       @log_size       = db.execute "SELECT count(datetime) from LogLine"
-      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @log_size       = @log_size[0][0]
+
       @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
+      @selfpolls_size = @selfpolls_size[0][0]
+
+      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @crawlers_size = @crawlers_size[0][0]
 
       @first_day_requested = options[:from_date]
       @last_day_requested = options[:to_date]
@@ -74,8 +79,13 @@ module LogSense
       EOS
 
       @total_hits = db.execute "SELECT count(datetime) from LogLine where #{filter}"
-      @total_unique_visitors = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{filter}"
+      @total_hits = @total_hits[0][0]
+
+      @total_unique_visits = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{filter}"
+      @total_unique_visits = @total_unique_visits[0][0]
+
       @total_size = db.execute "SELECT #{human_readable_size} from LogLine where #{filter}"
+      @total_size = @total_size[0][0]
 
       @daily_distribution = db.execute "SELECT date(datetime), #{human_readable_day}, count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
       @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"

data/lib/log_sense/emitter.rb

@@ -33,14 +33,8 @@ module LogSense
 
     private
 
-    def self.output_txt_table name, headings, rows
-      name = "#+NAME: #{name}"
-      table = Terminal::Table.new headings: headings, rows: rows, style: { border_x: "-", border_i: "|" }
-      name + "\n" + table.to_s
-    end
-
     def self.render(template, vars)
-      @template = File.join(File.dirname(__FILE__), "templates", "_#{template}.html.erb")
+      @template = File.join(File.dirname(__FILE__), "templates", "_#{template}")
       erb_template = File.read @template
       ERB.new(erb_template).result(OpenStruct.new(vars).instance_eval { binding })
     end

data/lib/log_sense/options_parser.rb

@@ -1,6 +1,6 @@
 require 'optparse'
 require 'optparse/date'
-require 'apache_log_report/version'
+require 'log_sense/version'
 
 module LogSense
   module OptionsParser
@@ -14,19 +14,19 @@ module LogSense
       opt_parser = OptionParser.new do |opts|
         opts.banner = "Usage: log_sense [options] [logfile]"
 
-        opts.on("-fFORMAT", "--from=FORMAT", String, "Input format (either rails or apache)") do |n|
+        opts.on("-fFORMAT", "--input-format=FORMAT", String, "Input format (either rails or apache)") do |n|
           args[:input_format] = n
         end
 
-        opts.on("-iINPUT_FILE", "--input=INPUT_FILE", String, "Input file") do |n|
+        opts.on("-iINPUT_FILE", "--input-file=INPUT_FILE", String, "Input file") do |n|
           args[:input_file] = n
         end
 
-        opts.on("-tFORMAT", "--to=FORMAT", String, "Output format: html, org, txt, sqlite. Defaults to org mode") do |n|
+        opts.on("-tFORMAT", "--output-format=FORMAT", String, "Output format: html, org, txt, sqlite. Defaults to org mode") do |n|
           args[:output_format] = n
         end
 
-        opts.on("-oOUTPUT_FILE", "--output=OUTPUT_FILE", String, "Output file") do |n|
+        opts.on("-oOUTPUT_FILE", "--output-file=OUTPUT_FILE", String, "Output file") do |n|
           args[:output_file] = n
         end
 
@@ -57,16 +57,28 @@ module LogSense
 
         opts.on("-v", "--version", "Prints version information") do
           puts "log_sense version #{LogSense::VERSION}"
-          puts "Copyright (C) 2020 Adolfo Villafiorita"
+          puts "Copyright (C) 2021 Shair.Tech"
           puts "Distributed under the terms of the MIT license"
-          puts ""
-          puts "Written by Adolfo Villafiorita"
           exit
         end
 
         opts.on("-h", "--help", "Prints this help") do
           puts opts
+          puts ""
           puts "This is version #{LogSense::VERSION}"
+
+          puts ""
+          puts "Output formats"
+          pathname = File.join(File.dirname(__FILE__), "templates", "*")
+          templates = Dir.glob(pathname).select { |x| ! File.basename(x).start_with? /_|#/ and ! File.basename(x).end_with? "~" }
+          components = templates.map { |x| File.basename(x).split "." }.group_by { |x| x[0] }
+          components.each do |k, vs|
+            puts "#{k} parsing can produce the following outputs:"
+            vs.each do |v|
+              puts "  - #{v[1]}"
+            end
+          end
+
           exit
         end
       end

data/lib/log_sense/rails_data_cruncher.rb

@@ -8,22 +8,28 @@ module LogSense
     #
 
     def self.crunch db, options = { limit: 30 }
-      first_day_s = db.execute "SELECT started_at from Event order by started_at limit 1"
+      first_day_s = db.execute "SELECT started_at from Event where started_at not NULL order by started_at limit 1"
       # we could use ended_at to cover the full activity period, but I prefer started_at
       # with the meaning that the monitor event initiation
       last_day_s  = db.execute "SELECT started_at from Event order by started_at desc limit 1"
 
       # make first and last day into dates or nil
       # TODO: bug possible value here: [[nil]], which is not empty
-      @first_day = first_day_s.empty? ? nil : Date.parse(first_day_s[0][0])
-      @last_day = last_day_s.empty? ? nil : Date.parse(last_day_s[0][0])
+      @first_day = ! first_day_s[0][0] ? nil : Date.parse(first_day_s[0][0])
+      @last_day = ! last_day_s[0][0] ? nil : Date.parse(last_day_s[0][0])
 
       @total_days = 0
       if @first_day and @last_day
         @total_days = (@last_day - @first_day).to_i
       end
 
+      @log_size = db.execute "SELECT count(started_at) from Event"
+      @log_size = @log_size[0][0]
+
+      # SAME AS ABOVE (but log_size is wrong in the case of Rails
+      # logs, since an event takes more than one line)
       @events = db.execute "SELECT count(started_at) from Event"
+      @events = @events[0][0]
 
       @first_day_requested = options[:from_date]
       @last_day_requested = options[:to_date]
@@ -74,6 +80,9 @@ module LogSense
 
       @total_events = db.execute "SELECT count(started_at) from Event where #{filter}"
 
+      @total_unique_visits = db.execute "SELECT count(distinct(unique_visitor)) from Event where #{filter}"
+      @total_unique_visits = @total_unique_visits[0][0]
+
       @daily_distribution = db.execute "SELECT date(started_at), #{human_readable_day}, count(started_at) from Event where #{filter} group by date(started_at)"
       @time_distribution = db.execute "SELECT strftime('%H', started_at), count(started_at) from Event where #{filter} group by strftime('%H', started_at)"
 

data/lib/log_sense/rails_log_parser.rb

@@ -12,6 +12,7 @@ module LogSense
          ended_at TEXT,
          log_id TEXT,
          ip TEXT,
+         unique_visitor TEXT,
          url TEXT,
          controller TEXT,
          html_verb TEXT,
@@ -27,6 +28,7 @@ module LogSense
          ended_at,
          log_id,
          ip,
+         unique_visitor,
          url,
          controller,
          html_verb,
@@ -35,7 +37,7 @@ module LogSense
          duration_views_ms,
          duration_ar_ms,
          allocations)
-      values (#{Array.new(12, '?').join(', ')})")
+      values (#{Array.new(13, '?').join(', ')})")
 
       # requests in the log might be interleaved.
       #
@@ -90,6 +92,7 @@ module LogSense
               event[:ended_at],
               event[:log_id],
               event[:ip],
+              "#{DateTime.parse(event[:ended_at]).strftime("%Y-%m-%d")} #{event[:ip]}",
               event[:url],
               event[:controller],
               event[:html_verb],

data/lib/log_sense/templates/_command_invocation.html.erb

@@ -0,0 +1,29 @@
+<table class="table unstriped command-invocation">
+  <tbody>
+    <tr>
+      <th>CLI Command</th>
+      <td><code><%= data[:command] %></code></td>
+    </tr>
+    <tr>
+      <th>Input file</th>
+      <td><code><%= (data[:log_file] || "stdin") %></code></td>
+    </tr>
+    <tr>
+      <th>Ignore crawlers</th>
+      <td><code><%= options[:ignore_crawlers] %></code></td></tr>
+    <tr>
+      <th>Only crawlers</th>
+      <td><code><%= options[:only_crawlers] %></code></td>
+    </tr>
+    <tr>
+      <th>No selfpoll</th>
+      <td><code><%= options[:no_selfpoll] %></code></td>
+    </tr>
+    <tr>
+      <th>Filter by date</th>
+      <td>
+        <code><%= (options[:from_date] != nil or options[:to_date] != nil) %></code>
+      </td>
+    </tr>
+  </tbody>
+</table>

data/lib/log_sense/templates/_command_invocation.txt.erb

@@ -0,0 +1,6 @@
+<%=
+table = Terminal::Table.new rows: [ ["Command", data[:command] ],
+                                    ["Input file", data[:log_file] || "stdin" ]
+                                  ]
+table
+%>

data/lib/log_sense/templates/_output_table.html.erb

@@ -4,7 +4,7 @@ def slugify string
 end
 %>
 
-<table id="<%= slugify(title || "") %>" class="table <%= slugify(title || "") %>">
+<table id="<%= slugify(title || "") %>-table" class="table unstriped data-table <%= slugify(title || "") %>">
   <thead>
     <tr>
       <% header.each do |heading| %>

data/lib/log_sense/templates/_performance.html.erb

@@ -0,0 +1,23 @@
+<table class="table unstriped performance">
+  <tbody>
+    <tr>
+      <th>Analysis started at</th>
+      <td><%= data[:started_at].to_s %></td>
+    </tr>
+    <tr>
+      <th>Analysis ended at</th>
+      <td><%= data[:ended_at].to_s %></td>
+    </tr>
+    <tr>
+      <th>Duration</th>
+      <td><%= "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] %></td>
+    </tr>
+    <tr>
+      <th>Events</th>
+      <td><%= data[:log_size] %></td>
+    </tr>
+    <tr>
+      <th>Parsed Events/sec</th>
+      <td><%= "%.2f" % (data[:log_size] / data[:duration]) %></td></tr>
+  </tbody>
+</table>

data/lib/log_sense/templates/_performance.txt.erb

@@ -0,0 +1,9 @@
+<%=
+table = Terminal::Table.new rows: [ ["Analysis started at", data[:started_at].to_s ],
+                                    ["Analysis ended at",   data[:ended_at].to_s ],
+                                    ["Duration",            "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] ],
+                                    ["Events",              "%9d" % data[:log_size] ],
+                                    ["Parsed events/sec",   "%.2f" % (data[:log_size] / data[:duration]) ] ]
+table.align_column(2, :right)
+table
+%>

data/lib/log_sense/templates/_summary.html.erb

@@ -0,0 +1,34 @@
+<table class="table unstriped summary">
+  <tr>
+    <th>Input file</th>
+    <td><b><%= (data[:log_file] || "stdin") %></b></td>
+  </tr>
+  <tr>
+    <th class="period">Period Analyzed</th>
+    <td class="period">
+      <%= data[:first_day_in_analysis] %>
+      --
+      <%= data[:last_day_in_analysis] %>
+    </td>
+  </tr>
+  <tr>
+    <th class="days">Days </th>
+    <td class="days"><%= data[:total_days_in_analysis] %></td>
+  </tr>
+  <tr>
+    <th class="hits">Hits</th>
+    <td class="hits"><%= data[:total_hits] %></td>
+  </tr>
+  <tr>
+    <th class="unique-visits">Unique Visits</th>
+    <td class="unique-visits"><%= data[:total_unique_visits] %></td>
+  </tr>
+  <tr>
+    <th class="avg-hits-per-unique-visits">Unique Visits</th>
+    <td class="avg-hits-per-unique-visits"><%= data[:total_hits] / data[:total_unique_visits] %></td>
+  </tr>
+  <tr>
+    <th class="tx">Tx</th>
+    <td class="tx"><%= data[:total_size] %></td>
+  </tr>
+</table>

data/lib/log_sense/templates/_summary.txt.erb

@@ -0,0 +1,10 @@
+<%=
+ table = Terminal::Table.new rows: [ ["Input File",      data[:log_file] || "stdin" ],
+                                     ["Period Analyzed", "#{data[:first_day_in_analysis]} -- #{data[:last_day_in_analysis]}" ],
+                                     ["Days",            data[:total_days_in_analysis] ],
+                                     ["Events",          data[:events] ],
+                                     ["Unique Visits",   data[:total_unique_visits] ],
+                                     ["Avg. Events per Visit", data[:events] / data[:total_unique_visits] ]
+                                   ]
+ table
+%>

data/lib/log_sense/templates/_total_hits.html.erb

@@ -0,0 +1,32 @@
+<table class="table unstriped log-structure">
+  <tbody>
+    <tr>
+      <th>Input file</th>
+      <td><b><%= (data[:log_file] || "stdin") %></b></td>
+    </tr>
+    <tr>
+      <th>Period in Log</th>
+      <td><%= data[:first_day] %> -- <%= data[:last_day] %></td>
+    </tr>
+    <tr>
+      <th>Total days</th>
+      <td><%= data[:total_days] %></td>
+    </tr>
+    <tr>
+      <th>Log size</th>
+      <td><%= data[:log_size] %></td>
+    </tr>
+    <tr>
+      <th>Self poll entries</th>
+      <td><%= data[:selfpolls_size] %></td>
+    </tr>
+    <tr>
+      <th>Crawlers</th>
+      <td><%= data[:crawlers_size] %></td>
+    </tr>
+    <tr>
+      <th>Entries considered</th>
+      <td><%= data[:total_hits] %></td>
+    </tr>
+  </tbody>
+</table>