log_sense 1.0.11 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0b958a6580aff4f478248d606897af928ffe23d6e8d8f43abcdeca50fbac6db
-  data.tar.gz: 9ecd13a65d80653e50b242938bce8083ff1e9ec590260fea3f8b5fc04b5e6a81
+  metadata.gz: d96a22ce71f0c0266811faa1853981d1211b93687bfd2e3077b91189feb5742b
+  data.tar.gz: 014ad1230a6a83b379310ba2b93b16bfd956db19a8f087bd2c216b5e77211620
 SHA512:
-  metadata.gz: 5a0c3a5e40a7e7cc74df707062fbd7cf958341696da9388b2d941f83fc219cdbcc8a258660ad85daa445bb781b3ca736350ac961907ff4220e6844b9c95bba55
-  data.tar.gz: e670fe25f1f8b368b77cb7259aac46fbb551751423ce322e50f5e48087703a01bdb5a08d242522db6a04693f1676cc7e123212d4ddf771260bd1febdec272835
+  metadata.gz: 8fe6fb1a329b0cabae06199f13c70936c9c7aeaa6c99142a683414ac380bfc23fecb14d3588803f2f9afc24483bfb8f429f867633ce5029c3dc23df1c713dea4
+  data.tar.gz: 10fe88f045845a5573f8ba7aedde8d3475a0f390e6025746799525be685c6db11e5eb9148e9de3f8f05a7950ae9c912f2d67e454a285de08143789db374818c7

data/CHANGELOG.org

@@ -2,26 +2,13 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
-* Unreleased
+* Changes in log_sense 1.1.2
+  <2021-12-17 Fri>
 
-This changes are in the repository but not yet released to Rubygems.
+- Added Rails Log HTML output
 
-** New Functions and Changes
+* Changes in log_sense 1.1.1 and earlier
+  <2021-12-17 Fri>
 
-** Fixes
-
-** Documentation
-
-** Code
-
-
-* Version 1.0.0
-
-** New Functions and Changes
-
-** Fixes
-
-** Documentation
-
-** Code
+- In the Git commit messages (not very informative, I am afraid).
 

data/README.org

@@ -14,21 +14,15 @@ and [[https://umami.is/][Umami]], focusing on privacy and data-ownership: the da
 generated by LogSense is stored on your computer and owned by
 you (like it should be).
 
-LogSense is also inspired by static websites generators:
-statistics are generated from the command line and accessed as static
-HTML files.  By generating static resources, LogSense
-significantly reduces the attack surface of your webserver and
-installation headaches.
+LogSense is also inspired by *static websites generators*: statistics
+are generated from the command line and accessed as static HTML files.
+By generating static resources, LogSense significantly reduces the
+attack surface of your webserver and installation headaches.
 
 We have, for instance, a cron job running on our servers, generating
 statistics at night.  The generated files are then made available on a
 private area on the web.
 
-Statistics are generated from Apache log formats in the =combined=
-format and from Rails logs.  Reports are tailored, but not limited, to
-web servers serving static websites.  No need to install Java Script
-code on your websites, no cookies installed, no user tracking.
-
 LogSense reports the following data:
 
 - Visitors, hits, unique visitors, bandwidth used
@@ -62,20 +56,29 @@ LogSense generates HTML, txt (Org Mode), and SQLite outputs.
 
   #+RESULTS:
   #+begin_example
-  Usage: apache_log_report [options] [logfile]
-      -l, --limit=N                    Number of entries to show (defaults to 30)
+  Usage: log_sense [options] [logfile]
+          --title=TITLE                Title to use in the report
+      -f, --input-format=FORMAT        Input format (either rails or apache)
+      -i, --input-file=INPUT_FILE      Input file
+      -t, --output-format=FORMAT       Output format: html, org, txt, sqlite. See below for available formats
+      -o, --output-file=OUTPUT_FILE    Output file
       -b, --begin=DATE                 Consider entries after or on DATE
       -e, --end=DATE                   Consider entries before or on DATE
-      -i, --ignore-crawlers            Ignore crawlers
-      -p, --ignore-selfpoll            Ignore apaches self poll entries (from ::1)
-          --only-crawlers              Perform analysis on crawlers only
-      -u, --prefix=PREFIX              Prefix to add to all plots (used to run multiple analyses in the same dir)
-      -w, --suffix=SUFFIX              Suffix to add to all plots (used to run multiple analyses in the same dir)
-      -c, --code-export=WHAT           Control :export directive in Org Mode code blocks (code, results, *both*, none)
-      -f, --format=FORMAT              Output format: html, org, sqlite. Defaults to org mode
+      -l, --limit=N                    Number of entries to show (defaults to 30)
+      -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
+      -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
-  This is version 1.1.6
+
+  This is version 1.1.1
+
+  Output formats
+  apache parsing can produce the following outputs:
+    - sqlite
+    - html
+  rails parsing can produce the following outputs:
+    - sqlite
+    - txt
   #+end_example
 
 * Change Log

data/lib/log_sense/emitter.rb

@@ -1,4 +1,5 @@
 require 'terminal-table'
+require 'json'
 require 'erb'
 require 'ostruct'
 

data/lib/log_sense/options_parser.rb

@@ -14,6 +14,10 @@ module LogSense
       opt_parser = OptionParser.new do |opts|
         opts.banner = "Usage: log_sense [options] [logfile]"
 
+        opts.on("-tTITLE", "--title=TITLE", String, "Title to use in the report") do |n|
+          args[:title] = n
+        end
+
         opts.on("-fFORMAT", "--input-format=FORMAT", String, "Input format (either rails or apache)") do |n|
           args[:input_format] = n
         end

data/lib/log_sense/rails_data_cruncher.rb

@@ -26,6 +26,10 @@ module LogSense
       @log_size = db.execute "SELECT count(started_at) from Event"
       @log_size = @log_size[0][0]
 
+      # TODO: I should make the names of events/size/etc uniform betweeen Apache and Rails Logs
+      # SAME AS ABOVE
+      @total_hits = @log_size
+
       # SAME AS ABOVE (but log_size is wrong in the case of Rails
       # logs, since an event takes more than one line)
       @events = db.execute "SELECT count(started_at) from Event"
@@ -88,11 +92,12 @@ module LogSense
 
       @statuses = db.execute "SELECT status, count(status) from Event where #{filter} group by status order by status"
 
+      @by_day_5xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '5' and #{filter} group by date(started_at)"
       @by_day_4xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '4' and #{filter} group by date(started_at)"
       @by_day_3xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '3' and #{filter} group by date(started_at)"
       @by_day_2xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '2' and #{filter} group by date(started_at)"
 
-      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
+      @statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx + @by_day_5xx).group_by { |x| x[0] }.to_a.map { |x|
         [x[0], x[1].map { |y| y[1] }].flatten
       }
 
@@ -100,8 +105,12 @@ module LogSense
 
       @performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
 
-      @fatal = db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, log_id FROM Event WHERE exit_status == 'F'"
+      @fatal = db.execute ("SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE exit_status == 'F'") || [[]]
+
+      @internal_server_error = (db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), status, ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE status is 500") || [[]]
 
+      @error = (db.execute "SELECT log_id, context, description, count(log_id) from Error GROUP BY description") || [[]]
+      
       data = {}
       self.instance_variables.each do |variable|
         var_as_symbol = variable.to_s[1..-1].to_sym

data/lib/log_sense/rails_log_parser.rb

@@ -1,4 +1,5 @@
 require 'sqlite3'
+require 'byebug'
 
 module LogSense
   module RailsLogParser
@@ -24,7 +25,7 @@ module LogSense
          allocations INTEGER,
          comment TEXT
       )'
-      
+
       ins = db.prepare("insert into Event(
          exit_status,
          started_at,
@@ -44,6 +45,22 @@ module LogSense
       )
       values (#{Array.new(15, '?').join(', ')})")
 
+      
+      db.execute 'CREATE TABLE IF NOT EXISTS Error(
+         id INTEGER PRIMARY KEY AUTOINCREMENT,
+         log_id TEXT,
+         context TEXT,
+         description TEXT
+      )'
+
+      ins_error = db.prepare("insert into Error(
+         log_id,
+         context,
+         description
+      )
+      values (?, ?, ?)")
+      
+
       # requests in the log might be interleaved.
       #
       # We use the 'pending' variable to progressively store data
@@ -65,8 +82,14 @@ module LogSense
       # Different requests might be interleaved, of course
       
       File.readlines(filename).each do |line|
-        # We discard LOG_LEVEL != 'I'
-        next if line[0] != 'I' and line[0] != 'F'
+        # I and F for completed requests, [ is for error messages
+        next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
+
+        data = self.match_and_process_error line
+        if data
+          ins_error.execute(data[:log_id], data[:context], data[:description])
+          next
+        end
         
         data = self.match_and_process_start line
         if data
@@ -145,39 +168,6 @@ module LogSense
           end
         end
 
-
-        data = self.match_and_process_completed_no_alloc line
-        if data
-          id = data[:log_id]
-
-          # it might as well be that the first event started before
-          # the log.  With this, we make sure we add only events whose
-          # start was logged and parsed
-          if pending[id]
-            event = data.merge (pending[id] || {})
-
-            ins.execute(
-              event[:exit_status],
-              event[:started_at],
-              event[:ended_at],
-              event[:log_id],
-              event[:ip],
-              "#{DateTime.parse(event[:ended_at]).strftime("%Y-%m-%d")} #{event[:ip]}",
-              event[:url],
-              event[:controller],
-              event[:html_verb],
-              event[:status],
-              event[:duration_total_ms],
-              event[:duration_views_ms],
-              event[:duration_ar_ms],
-              event[:allocations],
-              event[:comment]
-            )
-
-            pending.delete(id)
-          end
-        end
-
       end
       
       db
@@ -189,8 +179,30 @@ module LogSense
     URL = /(?<url>[^"]+)/
     IP = /(?<ip>[0-9.]+)/
     STATUS = /(?<status>[0-9]+)/
+    STATUS_IN_WORDS = /(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)/
     MSECS = /[0-9.]+/
 
+    # Error Messages
+    # [584cffcc-f1fd-4b5c-bb8b-b89621bd4921] ActionController::RoutingError (No route matches [GET] "/assets/foundation-icons.svg"):
+    # [fd8df8b5-83c9-48b5-a056-e5026e31bd5e] ActionView::Template::Error (undefined method `all_my_ancestor' for nil:NilClass):
+    # [d17ed55c-f5f1-442a-a9d6-3035ab91adf0] ActionView::Template::Error (undefined method `volunteer_for' for #<DonationsController:0x007f4864c564b8>
+    CONTEXT = /(?<context>[^ ]+Error)/
+    ERROR_REGEXP = /^\[#{ID}\] #{CONTEXT} \((?<description>.*)\):/
+
+    def self.match_and_process_error line
+      matchdata = ERROR_REGEXP.match line
+      if matchdata
+        {
+          log_id: matchdata[:id],
+          context: matchdata[:context],
+          description: matchdata[:description]
+        }
+      else
+        nil
+      end
+    end
+
+
     # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
     STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Started #{VERB} "#{URL}" for #{IP} at/
 
@@ -209,11 +221,15 @@ module LogSense
       end
     end
 
+    # TODO: Add regexps for the performance data (Views ...). We have three cases (view, active records, allocations), (views, active records), (active records, allocations)
     # I, [2021-10-19T08:16:34.712331 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
-    COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms \| Allocations: (?<alloc>[0-9]+)\)/
+    # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
+    # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
+    COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
 
     def self.match_and_process_completed line
       matchdata = (COMPLETED_REGEXP.match line)
+      # exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
       if matchdata
         {
           exit_status: "I",
@@ -231,29 +247,6 @@ module LogSense
       end
     end
 
-    # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
-    COMPLETED_NO_ALLOC_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms\)/
-
-    def self.match_and_process_completed_no_alloc line
-      matchdata = (COMPLETED_NO_ALLOC_REGEXP.match line)
-      if matchdata
-        {
-          exit_status: "I",
-          ended_at: matchdata[:timestamp],
-          log_id: matchdata[:id],
-          status: matchdata[:status],
-          duration_total_ms: matchdata[:total],
-          duration_views_ms: matchdata[:views],
-          duration_ar_ms: matchdata[:arec],
-          allocations: -1,
-          comment: ""
-        }
-      else
-        nil
-      end
-    end
-
-
     # I, [2021-10-19T08:16:34.345162 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
     PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{ID}\] Processing by (?<controller>[^ ]+) as/
 
@@ -289,7 +282,7 @@ module LogSense
     end
 
     # generate a unique visitor id from an event
-    def unique_visitor_id event
+    def self.unique_visitor_id event
       "#{DateTime.parse(event[:started_at] || event[:ended_at] || "1970-01-01").strftime("%Y-%m-%d")} #{event[:ip]}"
     end
 

data/lib/log_sense/templates/_log_structure.html.erb

@@ -0,0 +1,24 @@
+<ul class="stats-list">
+  <li>
+    <%= data[:first_day].strftime("%b %d, %Y") %>
+    <span class="stats-list-label">From</span>
+  </li>
+  <li>
+    <%= data[:last_day].strftime("%b %d, %Y") %>
+    <span class="stats-list-label">To</span>
+  </li>
+  <li class="stats-list-positive">
+    <%= data[:total_days] %> <span class="stats-list-label">Days in Log</span>
+  </li>
+  <li class="stats-list-negative">
+    <%= data[:log_size] %> <span class="stats-list-label">Total Entries</span>
+  </li>
+  <li class="stats-list-negative">
+    <%= data[:selfpolls_size] %> <span class="stats-list-label">Self Polls Entries</span>
+  </li>
+  <li class="stats-list-negative">
+    <td><%= data[:crawlers_size] %></td>
+    <span class="stats-list-label">Crawlers Entries</span>
+  </li>
+</ul>
+

data/lib/log_sense/templates/_performance.html.erb

@@ -1,23 +1,21 @@
-<table class="table unstriped performance">
-  <tbody>
-    <tr>
-      <th>Analysis started at</th>
-      <td><%= data[:started_at].to_s %></td>
-    </tr>
-    <tr>
-      <th>Analysis ended at</th>
-      <td><%= data[:ended_at].to_s %></td>
-    </tr>
-    <tr>
-      <th>Duration</th>
-      <td><%= "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] %></td>
-    </tr>
-    <tr>
-      <th>Events</th>
-      <td><%= data[:log_size] %></td>
-    </tr>
-    <tr>
-      <th>Parsed Events/sec</th>
-      <td><%= "%.2f" % (data[:log_size] / data[:duration]) %></td></tr>
-  </tbody>
-</table>
+<ul class="stats-list">
+  <li>
+    <%= data[:started_at].strftime("%b %d, %Y @ %H:%M:%S") %>
+    <span class="stats-list-label">Analysis Started</span>
+  </li>
+  <li>
+    <%= data[:ended_at].strftime("%b %d, %Y @ %H:%M:%S") %>
+    <span class="stats-list-label">Analysis Ended</span>
+  </li>
+  <li class="stats-list-negative">
+    <%= "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] %>
+    <span class="stats-list-label">Duration</span>
+  </li>
+  <li class="stats-list-negative">
+    <%= data[:log_size] %> <span class="stats-list-label">Events</span>
+  </li>
+  <li class="stats-list-positive">
+    <td><%= "%.2f" % (data[:log_size] / data[:duration]) %>
+    <span class="stats-list-label">Parsed Events/sec</span>
+  </li>
+</ul>

data/lib/log_sense/templates/_summary.html.erb

@@ -1,34 +1,23 @@
-<table class="table unstriped summary">
-  <tr>
-    <th>Input file</th>
-    <td><b><%= (data[:log_file] || "stdin") %></b></td>
-  </tr>
-  <tr>
-    <th class="period">Period Analyzed</th>
-    <td class="period">
-      <%= data[:first_day_in_analysis] %>
-      --
-      <%= data[:last_day_in_analysis] %>
-    </td>
-  </tr>
-  <tr>
-    <th class="days">Days </th>
-    <td class="days"><%= data[:total_days_in_analysis] %></td>
-  </tr>
-  <tr>
-    <th class="hits">Hits</th>
-    <td class="hits"><%= data[:total_hits] %></td>
-  </tr>
-  <tr>
-    <th class="unique-visits">Unique Visits</th>
-    <td class="unique-visits"><%= data[:total_unique_visits] %></td>
-  </tr>
-  <tr>
-    <th class="avg-hits-per-unique-visits">Unique Visits</th>
-    <td class="avg-hits-per-unique-visits"><%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %></td>
-  </tr>
-  <tr>
-    <th class="tx">Tx</th>
-    <td class="tx"><%= data[:total_size] %></td>
-  </tr>
-</table>
+<ul class="stats-list">
+  <li>
+    <%= data[:first_day_in_analysis].strftime("%b %d, %Y") %>
+    <span class="stats-list-label">From</span>
+  </li>
+  <li>
+    <%= data[:last_day_in_analysis].strftime("%b %d, %Y") %>
+    <span class="stats-list-label">To</span>
+  </li>
+  <li class="stats-list-positive">
+    <%= data[:total_days_in_analysis] %> <span class="stats-list-label">Days</span>
+  </li>
+  <li class="stats-list-negative">
+    <%= data[:total_hits] %> <span class="stats-list-label">Hits</span>
+  </li>
+  <li class="stats-list-negative">
+    <%= data[:total_unique_visits] %> <span class="stats-list-label">Unique Visits</span>
+  </li>
+  <li class="stats-list-negative">
+    <%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %>
+    <span class="stats-list-label">Unique Visits / Day</span>
+  </li>
+</ul>