log_sense 1.3.5 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30e8194103003ca9861272072bbf9ef199d7d80b67a0b73fb38d510f23adacee
-  data.tar.gz: 00b36829dd41b27e79cd6a41cacb412479631c0be9a9e2b79d585f81e0c7efa8
+  metadata.gz: 266e20972553f6d409814398dab832334a4c01bfa12e90c74acfdc75ee0b7c8d
+  data.tar.gz: 228e6bdc2d931e5190d82fc5ba66660ff6f0de0277a876746154de81a1ffe4e2
 SHA512:
-  metadata.gz: 41b392e7d5ec01052dbb645d50b6dc6736a82c04ea231ac33f3fa2c3679cc27a959164a8f5f9524f1604f1b495c032978a2e4e14ca01a706448fc6a8d7556185
-  data.tar.gz: fea27d4e0765fec9b090101a3efec3eb5ebd162af1c7389162d34c876754980b5dbccb22141b3aa4a0ec3eeb44bfd3936e17cc5463dc93148c3718bb51a1dcec
+  metadata.gz: f1454d78cfec258ff3bc69359be29178ebab4cf7ffd2869d736c29f2cffd6efe209f65be59298864bf94de30bd022a3397c91446b043c49e704b6d38ced59357
+  data.tar.gz: aa7239af4bb17270a23d9931194859b01a8ff50ebb9cc3c3ed37aeac1702aef413051b0381c57208dcffedd31ccbff5bd0a9485c0fe18947f540cda9f4463acd

data/CHANGELOG.org

@@ -2,6 +2,25 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.4.0
+
+- [User] The Apache Log report now organizes page requests in four
+  tables:
+  - success on HTML pages
+  - success on other resources
+  - failures on HTML pages
+  - failures on other resources
+- [User] Increased the default limit of pages in reports to 900
+- [User] The return status in now included in the page and resources
+  reports
+- [User] The "Attack" table has been removed, since the data can be
+  gotten from the previous tables
+- [Fixed] HTML pages are those with extension ".html" and ".htm"
+- [Fixed] Wrong data on summary table of the apache report has
+  been fixed
+- [Fixed] Better JavaScript escaping to avoid log poisoning
+- [Fixed] Strengthened the Apache log parser
+
 * 1.3.3 and 1.3.4
 
 - [Gem] Moved repository to Github and fixes to gemspec

data/lib/log_sense/apache_data_cruncher.rb

@@ -6,7 +6,7 @@ module LogSense
     # @ variables are automatically put in the returned data
     #
 
-    def self.crunch db, options = { limit: 30 }
+    def self.crunch db, options = { limit: 900 }
       first_day_s = db.execute "SELECT datetime from LogLine order by datetime limit 1"
       last_day_s  = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
 
@@ -89,16 +89,18 @@ module LogSense
 
       @daily_distribution = db.execute "SELECT date(datetime), #{human_readable_day}, count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
       @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"
-      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
-      @reasonable_requests_exts = [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].map {  |x|
-        "extension != '#{x}'"
-      }.join " and "
+      good_statuses = "(status like '2%' or status like '3%')"
+      bad_statuses = "(status like '4%' or status like '5%')"
+      html_page = "(extension like '.htm%')"
+      non_html_page = "(extension not like '.htm%')"
+
+      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{non_html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
-      @attacks = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} and (#{@reasonable_requests_exts}) group by path order by count(path) desc  limit #{options[:limit]}"
       @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
 
       @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"

data/lib/log_sense/apache_log_line_parser.rb

@@ -33,20 +33,19 @@ module LogSense
 
     HTTP_METHODS=/GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
     WEBDAV_METHODS=/COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-    OTHER_METHODS=/SEARCH|REPORT/
+    OTHER_METHODS=/SEARCH|REPORT|PRI|HEAD\/robots.txt/
     METHOD=/(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-    PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9])/
+    PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
     URL=/(?<url>[^ ]+)/
-    REFERER=/(?<referer>[^ ]+)/
+    REFERER=/(?<referer>[^"]*)/
     RETURN_CODE=/(?<status>[1-5][0-9][0-9])/
     SIZE=/(?<size>[0-9]+|-)/
-
-    USER_AGENT = /(?<user_agent>[^"]+)/
+    USER_AGENT = /(?<user_agent>[^"]*)/
 
     attr_reader :format
 
     def initialize 
-      @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "#{METHOD} #{URL} #{PROTOCOL}" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
+      @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
     end
 
     def parse line

data/lib/log_sense/emitter.rb

@@ -41,7 +41,20 @@ module LogSense
     end
 
     def self.escape_javascript(string)
-      js_escape_map = { "\\" => "\\\\", "</" => '<\/', "\r\n" => '\n', "\n" => '\n', "\r" => '\n', '"' => '\\"', "'" => "\\'", "`" => "\\`", "$" => "\\$" }
+      js_escape_map = {
+        "<script" => "&lt;script",
+        "</script" => "&lt;/script",
+        "<" => "&lt;",
+        "</" => '&lt;\/',
+        "\\" => "\\\\",
+        "\r\n" => '\\r\\n',
+        "\n" => '\\n',
+        "\r" => '\\r',
+        '"' => ' \\"',
+        "'" => " \\'",
+        "`" => " \\`",
+        "$" => " \\$"
+      }
       js_escape_map.each do |k, v|
         string = string.gsub(k, v)
       end

data/lib/log_sense/options_parser.rb

@@ -8,7 +8,7 @@ module LogSense
     # parse command line options
     #
     def self.parse options
-      limit = 30
+      limit = 900
       args = {} 
 
       opt_parser = OptionParser.new do |opts|

data/lib/log_sense/rails_data_cruncher.rb

@@ -7,7 +7,7 @@ module LogSense
     # @ variables are automatically put in the returned data
     #
 
-    def self.crunch db, options = { limit: 30 }
+    def self.crunch db, options = { limit: 900 }
       first_day_s = db.execute "SELECT started_at from Event where started_at not NULL order by started_at limit 1"
       # we could use ended_at to cover the full activity period, but I prefer started_at
       # with the meaning that the monitor event initiation

data/lib/log_sense/templates/_output_table.html.erb

@@ -19,6 +19,7 @@ end
  $(document).ready(function(){
    $('#table-<%= index %>').dataTable({
      data: data_<%= index %>,
+     <%= report[:datatable_options] + "," if report[:datatable_options] %>
      columns: [
        <% report[:header].each do |header| %>
        { data: '<%= header %>', className: '<%= slugify(header) %>' },

data/lib/log_sense/templates/_report_data.html.erb

@@ -4,7 +4,8 @@
    <% report[:rows].each do |row| %>
    {
      <% report[:header].each_with_index do |h, i| %>
-     "<%= h %>": <%= (row[i].class == Integer or row[i].class == Float) ? row[i] : "\"#{Emitter::escape_javascript(row[i] || '')}\"" %>,
+     <% resized_row = (row[i] || '').size > 150 ? "#{row[i][0..150]...}" : (row[i] || "") %>
+     "<%= h %>": <%= (row[i].class == Integer or row[i].class == Float) ? row[i] : "\"#{Emitter::escape_javascript(resized_row)}\"" %>,
      <% end %>
    },
    <% end %>

data/lib/log_sense/templates/_summary.html.erb

@@ -17,7 +17,12 @@
     <%= data[:total_unique_visits] %> <span class="stats-list-label">Unique Visits</span>
   </li>
   <li class="stats-list-negative">
-    <%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %>
+    <% days = data[:last_day_in_analysis] - data[:first_day_in_analysis] %>
+    <%= days > 0 ? "%.2f" % (data[:total_unique_visits] / days.to_f) : "N/A" %>
     <span class="stats-list-label">Unique Visits / Day</span>
   </li>
+  <li class="stats-list-negative">
+    <%= data[:total_unique_visits] != 0 ? data[:total_hits] / data[:total_unique_visits] : "N/A" %>
+    <span class="stats-list-label">Page Visited / Unique Visitor</span>
+  </li>
 </ul>

data/lib/log_sense/templates/apache.html.erb

@@ -157,11 +157,10 @@
                   "Log Structure",
                   "Daily Distribution",
                   "Time Distribution",
-                  "Most Requested Pages",
-                  "Most Requested Resources",
-                  "404 on HTML Files",
-                  "404 on other Resources",
-                  "Attacks",
+                  "20_ and 30_ on HTML pages",
+                  "20_ and 30_ on other resources",
+                  "40_ and 50_ on HTML pages",
+                  "40_ and 50_ on other Resources",
                   "Statuses",
                   "Daily Statuses",
                   "Browsers",
@@ -319,14 +318,26 @@
                 }
               }
             },
-            { title: "Most Requested Pages",
-              header: ["Path", "Hits", "Visits", "Size"],
+            { title: "20_ and 30_ on HTML pages",
+              header: ["Path", "Hits", "Visits", "Size", "Status"],
               rows: data[:most_requested_pages],
+              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
+            },
+            { title: "20_ and 30_ on other resources",
+              header: ["Path", "Hits", "Visits", "Size", "Status"],
+              rows: data[:most_requested_resources],
+              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
+            },
+            { title: "40_ and 50_x on HTML pages",
+              header: ["Path", "Hits", "Visits", "Status"],
+              rows: data[:missed_pages],
+              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
+            },
+            { title: "40_ and 50_ on other resources",
+              header: ["Path", "Hits", "Visits", "Status"],
+              rows: data[:missed_resources],
+              datatable_options: "columnDefs: [{ width: \"40%\", targets: 0 } ]"
             },
-            { title: "Most Requested Resources", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_resources] },
-            { title: "404 on HTML Files", header: ["Path", "Hits", "Visits"], rows: data[:missed_pages] },
-            { title: "404 on other Resources", header: ["Path", "Hits", "Visits"], rows: data[:missed_resources] },
-            { title: "Attacks", header: ["Path", "Hits", "Visits"], rows: data[:attacks], col: "small-12 cell" },
             { title: "Statuses",
               header: ["Status", "Count"],
               rows: data[:statuses],
@@ -494,10 +505,10 @@
                     <th>IP</th>
                     <th>
                       <div class="grid-x grid-margin-x">
-                        <div class="col-2 cell">
+                        <div class="small-2 cell">
                           Day
                         </div>
-                        <div class="col-10 cell">
+                        <div class="small-10 cell">
                           Resources
                         </div>
                       </div>
@@ -513,26 +524,25 @@
                       <td class="streaks">
                         <div class="grid-x grid-margin-x">
                           <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
-                            <div class="col-2 cell">
+                            <div class="small-12 medium-1 cell">
                               <span class="date"><%= date %></span>
                             </div>
-                            <div class="col-10 cell grid-x">
-                              <div class="small-12 medium-6 cell">
-                                <span class="res-title">HTML:</span>
-                                <ul>
-                                  <% urls.map { |x| x[2] }.compact.select { |x| x.match /.*\.html?/ }.each do |url| %>
-                                    <li><%= url %></li>
-                                  <% end %>
-                                </ul>
-                              </div>                              
-                              <div class=" small-12 medium-6 cell">
-                                <span class="res-title small-12 medium-6 cell">Other Resources:</span>
-                                <ul>
-                                  <% urls.map { |x| x[2] }.compact.sort.select { |x| x and not x.match /.*\.html?/ }.each do |url| %>
-                                    <li><%= url %></li>
-                                  <% end %>
-                                </ul>
-                              </div>
+                            <div class="small-12 medium-5 cell">
+                              <span class="res-title">HTML:</span>
+                              <% unique_with_count = urls.map { |x| x[2] }.compact.group_by{|e| e}.map{|k, v| [k, v.length]} %>
+                              <ul class="no-bullet">
+                                <% unique_with_count.select { |x| x[0].match /.*\.html?/ }.each do |url| %>
+                                  <li>[<%= url[1] %>] <%= Emitter::escape_javascript url[0] %></li>
+                                <% end %>
+                              </ul>
+                            </div>                              
+                            <div class=" small-12 medium-5 cell">
+                              <span class="res-title">Other Resources:</span>
+                              <ul class="no-bullet">
+                                <% unique_with_count.select { |x| x[0] and ! x[0].match /.*\.html?/ }.each do |url| %>
+                                  <li>[<%= url[1] %>] <%= Emitter::escape_javascript url[0] %></li>
+                                <% end %>
+                              </ul>
                             </div>
                           <% end %>
                         </div>

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "1.3.5"
+  VERSION = "1.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 1.3.5
+  version: 1.4.0
 platform: ruby
 authors:
 - Adolfo Fibrillation
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-01-26 00:00:00.000000000 Z
+date: 2022-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser