log_sense 1.3.4 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8250028b7a7038e07f2bd9fa069bfdf0ab1815a8711a0b6971a91030d529882
-  data.tar.gz: 236e1ba2c3ada9272f1e7eaa649cd1664776da933a90e41b6e70fffbbe002043
+  metadata.gz: 6ef1d35932ba8c7fe6e636f6cb507ea99fd6b6026170d62207dfa2606d62bbcb
+  data.tar.gz: 725dc6e51ace6c9cbd366e3888387ef8f246327c9e1036c66d2eb6351b1f0791
 SHA512:
-  metadata.gz: 72b6284cd6f09ebf16c4fe34fc7098da44e993a2946fb09f72ccaa7898eb462b9fcb49ce2e37e06b206a3f00650f8a033ec034ffeeab3afe587b6b607d079302
-  data.tar.gz: 6b96590430caa4e9717180c1812b0227b661f1fef4c99e10515da6da8acbe8fd596b91a9e0d536eff2ce525ba1fc01f4819a5c3f649bb59f4c77754edbbd5b0c
+  metadata.gz: 592e80cd56f4740cc4003b494c9da960de228025377d630d070a08a495cffac7fd41850d469189510def48f9940700251bd84a141def9b1b15b32fbed967261f
+  data.tar.gz: 1d802e95fd58c66c4904843478c5ccc8ffc1ce43e87dd199e5b21efc89bcff436b227c646d2e5a386bee22a3d1c67c0341c25fcb4058e48526560d05984f0148

data/CHANGELOG.org

@@ -2,6 +2,41 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.4.1
+
+- [User] New textual report for Apache
+- [User] New option -w sets maximum width of URL, Path, and
+  Description columns in textual reports
+- [User] Removed option -i, since input filenames are now taken
+  as direct arguments
+- [User] Allow multiple files in input
+- [Fixed] Complain if input format is not supported
+- [Code] Refactoring of reports to manage better output to
+  multiple formats  
+
+* 1.4.0
+
+- [User] The Apache Log report now organizes page requests in four
+  tables:
+  - success on HTML pages
+  - success on other resources
+  - failures on HTML pages
+  - failures on other resources
+- [User] Increased the default limit of pages in reports to 900
+- [User] The return status in now included in the page and resources
+  reports
+- [User] The "Attack" table has been removed, since the data can be
+  gotten from the previous tables
+- [Fixed] HTML pages are those with extension ".html" and ".htm"
+- [Fixed] Wrong data on summary table of the apache report has
+  been fixed
+- [Fixed] Better JavaScript escaping to avoid log poisoning
+- [Fixed] Strengthened the Apache log parser
+
+* 1.3.3 and 1.3.4
+
+- [Gem] Moved repository to Github and fixes to gemspec
+
 * 1.3.2
 
 - [Code] HTML reports now generate JSON data which is shared between

data/README.org

@@ -62,21 +62,21 @@ generated files are then made available on a private area on the web.
 
   #+RESULTS:
   #+begin_example
-  Usage: log_sense [options] [logfile]
+  Usage: log_sense [options] [logfile ...]
           --title=TITLE                Title to use in the report
       -f, --input-format=FORMAT        Input format (either rails or apache)
-      -i, --input-file=INPUT_FILE      Input file
       -t, --output-format=FORMAT       Output format: html, org, txt, sqlite. See below for available formats
       -o, --output-file=OUTPUT_FILE    Output file
       -b, --begin=DATE                 Consider entries after or on DATE
       -e, --end=DATE                   Consider entries before or on DATE
-      -l, --limit=N                    Number of entries to show (defaults to 30)
+      -l, --limit=N                    Limit to the N most requested resources (defaults to 900)
+      -w, --width=WIDTH                Maximum width of URL and description columns in text reports
       -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
       -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.3.1
+  This is version 1.4.1
 
   Output formats
   rails parsing can produce the following outputs:
@@ -85,6 +85,7 @@ generated files are then made available on a private area on the web.
     - html
   apache parsing can produce the following outputs:
     - sqlite
+    - txt
     - html
   #+end_example
 

data/exe/log_sense

@@ -7,21 +7,15 @@ require 'log_sense.rb'
 #
 
 # this better be here... OptionsParser consumes ARGV
-@command_line = ARGV.join(" ")
-
+@command_line = ARGV.join(' ')
 @options     = LogSense::OptionsParser.parse ARGV
-@input_file  = @options[:input_file] || ARGV[0]
 @output_file = @options[:output_file]
 
-if not @input_file
-  puts "Error: no input file specified."
-  exit
-end
-
-if not File.exist? @input_file
-  puts "Error: input file '#{@input_file}' does not exist"
+if ARGV.map { |x| File.exist?(x) }.include?(false)
+  warn.puts "Error: input file(s) '#{ARGV.reject { |x| File.exist(x) }.join(', ')}' do not exist"
   exit 1
 end
+@input_files = ARGV.empty? ? [$stdin] : ARGV.map { |x| File.open(x, 'r') }
 
 #
 # Parse Log and Track Statistics
@@ -36,12 +30,15 @@ when 'apache'
 when 'rails'
   parser_klass = LogSense::RailsLogParser
   cruncher_klass = LogSense::RailsDataCruncher
+else
+  warn.puts "Error: input format #{@options[:input_format]} not understood."
+  exit 1
 end
 
-@db = parser_klass.parse @input_file
+@db = parser_klass.parse @input_files
 
-if @options[:output_format]  == "sqlite"
-  ddb = SQLite3::Database.new(@output_file || "db.sqlite3")
+if @options[:output_format] == 'sqlite'
+  ddb = SQLite3::Database.new(@output_file || 'db.sqlite3')
   b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
   b.step(-1) #=> DONE
   b.finish
@@ -54,10 +51,11 @@ else
 
   @data = @data.merge({
                         command: @command_line,
-                        log_file: @input_file,
+                        log_files: @input_files,
                         started_at: @started_at,
                         ended_at: @ended_at,
-                        duration: @duration
+                        duration: @duration,
+                        width: @options[:width]
                       })
 
   #

data/lib/log_sense/apache_data_cruncher.rb

@@ -6,7 +6,7 @@ module LogSense
     # @ variables are automatically put in the returned data
     #
 
-    def self.crunch db, options = { limit: 30 }
+    def self.crunch db, options = { limit: 900 }
       first_day_s = db.execute "SELECT datetime from LogLine order by datetime limit 1"
       last_day_s  = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
 
@@ -15,9 +15,9 @@ module LogSense
       @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
 
       @total_days = 0
-      if @first_day and @last_day
-        @total_days = (@last_day - @first_day).to_i
-      end
+      @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
+
+      @source_files   = db.execute "SELECT distinct(source_file) from LogLine"
 
       @log_size       = db.execute "SELECT count(datetime) from LogLine"
       @log_size       = @log_size[0][0]
@@ -89,16 +89,18 @@ module LogSense
 
       @daily_distribution = db.execute "SELECT date(datetime), #{human_readable_day}, count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
       @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"
-      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
-      @reasonable_requests_exts = [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].map {  |x|
-        "extension != '#{x}'"
-      }.join " and "
+      good_statuses = "(status like '2%' or status like '3%')"
+      bad_statuses = "(status like '4%' or status like '5%')"
+      html_page = "(extension like '.htm%')"
+      non_html_page = "(extension not like '.htm%')"
+
+      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{non_html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
-      @attacks = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} and (#{@reasonable_requests_exts}) group by path order by count(path) desc  limit #{options[:limit]}"
       @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
 
       @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"

data/lib/log_sense/apache_log_line_parser.rb

@@ -31,22 +31,21 @@ module LogSense
 
     TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
 
-    HTTP_METHODS=/GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
-    WEBDAV_METHODS=/COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-    OTHER_METHODS=/SEARCH|REPORT/
-    METHOD=/(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-    PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9])/
-    URL=/(?<url>[^ ]+)/
-    REFERER=/(?<referer>[^ ]+)/
-    RETURN_CODE=/(?<status>[1-5][0-9][0-9])/
-    SIZE=/(?<size>[0-9]+|-)/
-
-    USER_AGENT = /(?<user_agent>[^"]+)/
+    HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
+    WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
+    OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
+    METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
+    PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
+    URL = /(?<url>[^ ]+)/
+    REFERER = /(?<referer>[^"]*)/
+    RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
+    SIZE = /(?<size>[0-9]+|-)/
+    USER_AGENT = /(?<user_agent>[^"]*)/
 
     attr_reader :format
 
     def initialize 
-      @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "#{METHOD} #{URL} #{PROTOCOL}" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
+      @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
     end
 
     def parse line

data/lib/log_sense/apache_log_parser.rb

@@ -7,10 +7,9 @@ module LogSense
     # parse an Apache log file and return a SQLite3 DB
     #
 
-    def self.parse filename, options = {}
-      content = filename ? File.readlines(filename) : ARGF.readlines
+    def self.parse(streams, options = {})
+      db = SQLite3::Database.new ':memory:'
 
-      db = SQLite3::Database.new ":memory:"
       db.execute "CREATE TABLE IF NOT EXISTS LogLine(
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       datetime TEXT,
@@ -28,15 +27,18 @@ module LogSense
       browser TEXT,
       browser_version TEXT,
       platform TEXT,
-      platform_version TEXT)"
+      platform_version TEXT,
+      source_file TEXT,
+      line_number INTEGER
+      )"
       
-      ins = db.prepare('insert into LogLine (
-                datetime, 
+      ins = db.prepare("insert into LogLine (
+                datetime,
                 ip,
                 user,
                 unique_visitor,
                 method,
-                path, 
+                path,
                 extension,
                 status,
                 size,
@@ -46,44 +48,50 @@ module LogSense
                 browser,
                 browser_version,
                 platform,
-                platform_version)
-              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+                platform_version,
+                source_file,
+                line_number
+                )
+              values (#{Array.new(18, '?').join(', ')})")
 
       parser = ApacheLogLineParser.new
-      
-      content.each do |line|
-        begin
-          hash = parser.parse line
-          ua = Browser.new(hash[:user_agent], accept_language: "en-us")
-          ins.execute(
-            DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
-            hash[:ip],
-            hash[:userid],
-            unique_visitor_id(hash),
-            hash[:method],
-            hash[:url],
-            (hash[:url] ? File.extname(hash[:url]) : ""),
-            hash[:status],
-            hash[:size].to_i,
-            hash[:referer],
-            hash[:user_agent],
-            ua.bot? ? 1 : 0,
-            (ua.name || ""),
-            (ua.version || ""),
-            (ua.platform.name || ""),
-            (ua.platform.version || "")
-          )
-        rescue StandardError => e
-          STDERR.puts e.message
+
+      streams.each do |stream|
+        stream.readlines.each_with_index do |line, line_number|
+          begin
+            hash = parser.parse line
+            ua = Browser.new(hash[:user_agent], accept_language: 'en-us')
+            ins.execute(
+              DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
+              hash[:ip],
+              hash[:userid],
+              unique_visitor_id(hash),
+              hash[:method],
+              hash[:url],
+              (hash[:url] ? File.extname(hash[:url]) : ''),
+              hash[:status],
+              hash[:size].to_i,
+              hash[:referer],
+              hash[:user_agent],
+              ua.bot? ? 1 : 0,
+              (ua.name || ''),
+              (ua.version || ''),
+              (ua.platform.name || ''),
+              (ua.platform.version || ''),
+              stream == $stdin ? "stdin" : stream.path,
+              line_number
+            )
+          rescue StandardError => e
+            warn.puts e.message
+          end
         end
       end
-      
+
       db
     end
 
     def self.unique_visitor_id hash
       "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
     end
-
   end
 end