log_sense 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 266e20972553f6d409814398dab832334a4c01bfa12e90c74acfdc75ee0b7c8d
-  data.tar.gz: 228e6bdc2d931e5190d82fc5ba66660ff6f0de0277a876746154de81a1ffe4e2
+  metadata.gz: 6ef1d35932ba8c7fe6e636f6cb507ea99fd6b6026170d62207dfa2606d62bbcb
+  data.tar.gz: 725dc6e51ace6c9cbd366e3888387ef8f246327c9e1036c66d2eb6351b1f0791
 SHA512:
-  metadata.gz: f1454d78cfec258ff3bc69359be29178ebab4cf7ffd2869d736c29f2cffd6efe209f65be59298864bf94de30bd022a3397c91446b043c49e704b6d38ced59357
-  data.tar.gz: aa7239af4bb17270a23d9931194859b01a8ff50ebb9cc3c3ed37aeac1702aef413051b0381c57208dcffedd31ccbff5bd0a9485c0fe18947f540cda9f4463acd
+  metadata.gz: 592e80cd56f4740cc4003b494c9da960de228025377d630d070a08a495cffac7fd41850d469189510def48f9940700251bd84a141def9b1b15b32fbed967261f
+  data.tar.gz: 1d802e95fd58c66c4904843478c5ccc8ffc1ce43e87dd199e5b21efc89bcff436b227c646d2e5a386bee22a3d1c67c0341c25fcb4058e48526560d05984f0148

data/CHANGELOG.org

@@ -2,6 +2,18 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.4.1
+
+- [User] New textual report for Apache
+- [User] New option -w sets maximum width of URL, Path, and
+  Description columns in textual reports
+- [User] Removed option -i, since input filenames are now taken
+  as direct arguments
+- [User] Allow multiple files in input
+- [Fixed] Complain if input format is not supported
+- [Code] Refactoring of reports to manage better output to
+  multiple formats  
+
 * 1.4.0
 
 - [User] The Apache Log report now organizes page requests in four

data/README.org

@@ -62,21 +62,21 @@ generated files are then made available on a private area on the web.
 
   #+RESULTS:
   #+begin_example
-  Usage: log_sense [options] [logfile]
+  Usage: log_sense [options] [logfile ...]
           --title=TITLE                Title to use in the report
       -f, --input-format=FORMAT        Input format (either rails or apache)
-      -i, --input-file=INPUT_FILE      Input file
       -t, --output-format=FORMAT       Output format: html, org, txt, sqlite. See below for available formats
       -o, --output-file=OUTPUT_FILE    Output file
       -b, --begin=DATE                 Consider entries after or on DATE
       -e, --end=DATE                   Consider entries before or on DATE
-      -l, --limit=N                    Number of entries to show (defaults to 30)
+      -l, --limit=N                    Limit to the N most requested resources (defaults to 900)
+      -w, --width=WIDTH                Maximum width of URL and description columns in text reports
       -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
       -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.3.1
+  This is version 1.4.1
 
   Output formats
   rails parsing can produce the following outputs:
@@ -85,6 +85,7 @@ generated files are then made available on a private area on the web.
     - html
   apache parsing can produce the following outputs:
     - sqlite
+    - txt
     - html
   #+end_example
 

data/exe/log_sense

@@ -7,21 +7,15 @@ require 'log_sense.rb'
 #
 
 # this better be here... OptionsParser consumes ARGV
-@command_line = ARGV.join(" ")
-
+@command_line = ARGV.join(' ')
 @options     = LogSense::OptionsParser.parse ARGV
-@input_file  = @options[:input_file] || ARGV[0]
 @output_file = @options[:output_file]
 
-if not @input_file
-  puts "Error: no input file specified."
-  exit
-end
-
-if not File.exist? @input_file
-  puts "Error: input file '#{@input_file}' does not exist"
+if ARGV.map { |x| File.exist?(x) }.include?(false)
+  warn.puts "Error: input file(s) '#{ARGV.reject { |x| File.exist(x) }.join(', ')}' do not exist"
   exit 1
 end
+@input_files = ARGV.empty? ? [$stdin] : ARGV.map { |x| File.open(x, 'r') }
 
 #
 # Parse Log and Track Statistics
@@ -36,12 +30,15 @@ when 'apache'
 when 'rails'
   parser_klass = LogSense::RailsLogParser
   cruncher_klass = LogSense::RailsDataCruncher
+else
+  warn.puts "Error: input format #{@options[:input_format]} not understood."
+  exit 1
 end
 
-@db = parser_klass.parse @input_file
+@db = parser_klass.parse @input_files
 
-if @options[:output_format]  == "sqlite"
-  ddb = SQLite3::Database.new(@output_file || "db.sqlite3")
+if @options[:output_format] == 'sqlite'
+  ddb = SQLite3::Database.new(@output_file || 'db.sqlite3')
   b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
   b.step(-1) #=> DONE
   b.finish
@@ -54,10 +51,11 @@ else
 
   @data = @data.merge({
                         command: @command_line,
-                        log_file: @input_file,
+                        log_files: @input_files,
                         started_at: @started_at,
                         ended_at: @ended_at,
-                        duration: @duration
+                        duration: @duration,
+                        width: @options[:width]
                       })
 
   #

data/lib/log_sense/apache_data_cruncher.rb

@@ -15,9 +15,9 @@ module LogSense
       @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
 
       @total_days = 0
-      if @first_day and @last_day
-        @total_days = (@last_day - @first_day).to_i
-      end
+      @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
+
+      @source_files   = db.execute "SELECT distinct(source_file) from LogLine"
 
       @log_size       = db.execute "SELECT count(datetime) from LogLine"
       @log_size       = @log_size[0][0]

data/lib/log_sense/apache_log_line_parser.rb

@@ -31,15 +31,15 @@ module LogSense
 
     TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
 
-    HTTP_METHODS=/GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
-    WEBDAV_METHODS=/COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-    OTHER_METHODS=/SEARCH|REPORT|PRI|HEAD\/robots.txt/
-    METHOD=/(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-    PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
-    URL=/(?<url>[^ ]+)/
-    REFERER=/(?<referer>[^"]*)/
-    RETURN_CODE=/(?<status>[1-5][0-9][0-9])/
-    SIZE=/(?<size>[0-9]+|-)/
+    HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
+    WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
+    OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
+    METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
+    PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
+    URL = /(?<url>[^ ]+)/
+    REFERER = /(?<referer>[^"]*)/
+    RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
+    SIZE = /(?<size>[0-9]+|-)/
     USER_AGENT = /(?<user_agent>[^"]*)/
 
     attr_reader :format

data/lib/log_sense/apache_log_parser.rb

@@ -7,10 +7,9 @@ module LogSense
     # parse an Apache log file and return a SQLite3 DB
     #
 
-    def self.parse filename, options = {}
-      content = filename ? File.readlines(filename) : ARGF.readlines
+    def self.parse(streams, options = {})
+      db = SQLite3::Database.new ':memory:'
 
-      db = SQLite3::Database.new ":memory:"
       db.execute "CREATE TABLE IF NOT EXISTS LogLine(
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       datetime TEXT,
@@ -28,15 +27,18 @@ module LogSense
       browser TEXT,
       browser_version TEXT,
       platform TEXT,
-      platform_version TEXT)"
+      platform_version TEXT,
+      source_file TEXT,
+      line_number INTEGER
+      )"
       
-      ins = db.prepare('insert into LogLine (
-                datetime, 
+      ins = db.prepare("insert into LogLine (
+                datetime,
                 ip,
                 user,
                 unique_visitor,
                 method,
-                path, 
+                path,
                 extension,
                 status,
                 size,
@@ -46,44 +48,50 @@ module LogSense
                 browser,
                 browser_version,
                 platform,
-                platform_version)
-              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+                platform_version,
+                source_file,
+                line_number
+                )
+              values (#{Array.new(18, '?').join(', ')})")
 
       parser = ApacheLogLineParser.new
-      
-      content.each do |line|
-        begin
-          hash = parser.parse line
-          ua = Browser.new(hash[:user_agent], accept_language: "en-us")
-          ins.execute(
-            DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
-            hash[:ip],
-            hash[:userid],
-            unique_visitor_id(hash),
-            hash[:method],
-            hash[:url],
-            (hash[:url] ? File.extname(hash[:url]) : ""),
-            hash[:status],
-            hash[:size].to_i,
-            hash[:referer],
-            hash[:user_agent],
-            ua.bot? ? 1 : 0,
-            (ua.name || ""),
-            (ua.version || ""),
-            (ua.platform.name || ""),
-            (ua.platform.version || "")
-          )
-        rescue StandardError => e
-          STDERR.puts e.message
+
+      streams.each do |stream|
+        stream.readlines.each_with_index do |line, line_number|
+          begin
+            hash = parser.parse line
+            ua = Browser.new(hash[:user_agent], accept_language: 'en-us')
+            ins.execute(
+              DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
+              hash[:ip],
+              hash[:userid],
+              unique_visitor_id(hash),
+              hash[:method],
+              hash[:url],
+              (hash[:url] ? File.extname(hash[:url]) : ''),
+              hash[:status],
+              hash[:size].to_i,
+              hash[:referer],
+              hash[:user_agent],
+              ua.bot? ? 1 : 0,
+              (ua.name || ''),
+              (ua.version || ''),
+              (ua.platform.name || ''),
+              (ua.platform.version || ''),
+              stream == $stdin ? "stdin" : stream.path,
+              line_number
+            )
+          rescue StandardError => e
+            warn.puts e.message
+          end
         end
       end
-      
+
       db
     end
 
     def self.unique_visitor_id hash
       "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
     end
-
   end
 end