log_sense 1.3.5 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30e8194103003ca9861272072bbf9ef199d7d80b67a0b73fb38d510f23adacee
-  data.tar.gz: 00b36829dd41b27e79cd6a41cacb412479631c0be9a9e2b79d585f81e0c7efa8
+  metadata.gz: '0128717b2ba709bc5dfbb7b755762a757c575ad4307159910cc894aaa3b88f42'
+  data.tar.gz: e05054b8eee79a439f5b077e60bc0d95a3e7706c550853333d7d631c458abf91
 SHA512:
-  metadata.gz: 41b392e7d5ec01052dbb645d50b6dc6736a82c04ea231ac33f3fa2c3679cc27a959164a8f5f9524f1604f1b495c032978a2e4e14ca01a706448fc6a8d7556185
-  data.tar.gz: fea27d4e0765fec9b090101a3efec3eb5ebd162af1c7389162d34c876754980b5dbccb22141b3aa4a0ec3eeb44bfd3936e17cc5463dc93148c3718bb51a1dcec
+  metadata.gz: 9d9e3dc495f7479292ae96d1bf6298f531258cae74df47ff705aea9613880b0d50aa6a19328f70685484ad1c606bd12a8fb7c87632fe5c9cbefefe4893d9bb4d
+  data.tar.gz: b417049bcc119ed82ab4c33d007e15804ba85b2485308ca28448fba512814fc5e8b8b215310bfb5c8108fcdc87292322eefc6826b18718fcbc7fced29eea77cb

data/CHANGELOG.org

@@ -2,6 +2,52 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 1.5.0
+
+- [User] Present Unique Visits / day as integer
+- [User] Added Country and Streaks report for rails
+- [User] Changed Streak report in Apache
+
+- [Gem] Updated DBIP
+- [Gem] Updated Bundle  
+
+- [Code] Refactored all reports, so that they are specified
+  in the same way  
+- [Code] Refactor warning message in textual reports
+- [Code] Build HTML menu for report specification
+- [Code] Various refactoring passes on the code
+
+* 1.4.1
+
+- [User] New textual report for Apache
+- [User] New option -w sets maximum width of URL, Path, and
+  Description columns in textual reports
+- [User] Removed option -i, since input filenames are now taken
+  as direct arguments
+- [User] Allow multiple files in input
+- [Fixed] Complain if input format is not supported
+- [Code] Refactoring of reports to manage better output to
+  multiple formats  
+
+* 1.4.0
+
+- [User] The Apache Log report now organizes page requests in four
+  tables:
+  - success on HTML pages
+  - success on other resources
+  - failures on HTML pages
+  - failures on other resources
+- [User] Increased the default limit of pages in reports to 900
+- [User] The return status in now included in the page and resources
+  reports
+- [User] The "Attack" table has been removed, since the data can be
+  gotten from the previous tables
+- [Fixed] HTML pages are those with extension ".html" and ".htm"
+- [Fixed] Wrong data on summary table of the apache report has
+  been fixed
+- [Fixed] Better JavaScript escaping to avoid log poisoning
+- [Fixed] Strengthened the Apache log parser
+
 * 1.3.3 and 1.3.4
 
 - [Gem] Moved repository to Github and fixes to gemspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (1.3.1)
+    log_sense (1.4.2)
       browser
       ipaddr
       iso_country_codes
@@ -13,9 +13,9 @@ GEM
   specs:
     browser (5.3.1)
     byebug (11.1.3)
-    ipaddr (1.2.3)
+    ipaddr (1.2.4)
     iso_country_codes (0.7.8)
-    minitest (5.14.4)
+    minitest (5.15.0)
     rake (12.3.3)
     sqlite3 (1.4.2)
     terminal-table (3.0.2)
@@ -32,4 +32,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.2.32
+   2.3.3

data/README.org

@@ -19,8 +19,6 @@ LogSense reports the following data:
 - OS, browsers, and devices
 - IP Country location, thanks to the DPIP lite country DB
 - Streaks: resources accessed by a given IP over time
-- Potential attacks: access to resources which are not meant to be
-  served by a web server serving static websites
 - Performance of Rails requests
  
 Filters from the command line allow to analyze specific periods and
@@ -33,6 +31,18 @@ And, of course, the compulsory screenshot:
 #+ATTR_HTML: :width 80%
 [[file:./apache-screenshot.png]]
 
+
+* An important word of warning
+
+[[https://owasp.org/www-community/attacks/Log_Injection][Log poisoning]] is a technique whereby attackers send requests with invalidated
+user input to forge log entries or inject malicious content into the logs.
+
+log_sense sanitizes entries of HTML reports, to try and protect from log
+poisoning.  *Log entries and URLs in SQLite3, however, are not sanitized*:
+they are stored and read from the log.  This is not, in general, an issue,
+unless you use the data from SQLite in environments in which URLs can be
+opened or code executed.
+
 * Motivation
 
 LogSense moves along the lines of tools such as [[https://goaccess.io/][GoAccess]] (which
@@ -54,6 +64,7 @@ generated files are then made available on a private area on the web.
   gem install log_sense
   #+end_src
 
+
 * Usage
 
   #+begin_src bash :results raw output :wrap example
@@ -62,21 +73,22 @@ generated files are then made available on a private area on the web.
 
   #+RESULTS:
   #+begin_example
-  Usage: log_sense [options] [logfile]
+  Usage: log_sense [options] [logfile ...]
           --title=TITLE                Title to use in the report
       -f, --input-format=FORMAT        Input format (either rails or apache)
-      -i, --input-file=INPUT_FILE      Input file
       -t, --output-format=FORMAT       Output format: html, org, txt, sqlite. See below for available formats
       -o, --output-file=OUTPUT_FILE    Output file
       -b, --begin=DATE                 Consider entries after or on DATE
       -e, --end=DATE                   Consider entries before or on DATE
-      -l, --limit=N                    Number of entries to show (defaults to 30)
+      -l, --limit=N                    Limit to the N most requested resources (defaults to 900)
+      -w, --width=WIDTH                Maximum width of URL and description columns in text reports
       -c, --crawlers=POLICY            Decide what to do with crawlers (applies to Apache Logs)
       -n, --no-selfpolls               Ignore self poll entries (requests from ::1; applies to Apache Logs)
+          --verbose                    Inform about progress (prints to STDERR)
       -v, --version                    Prints version information
       -h, --help                       Prints this help
 
-  This is version 1.3.1
+  This is version 1.5.0
 
   Output formats
   rails parsing can produce the following outputs:
@@ -85,6 +97,7 @@ generated files are then made available on a private area on the web.
     - html
   apache parsing can produce the following outputs:
     - sqlite
+    - txt
     - html
   #+end_example
 
@@ -95,6 +108,7 @@ log_sense -f apache -i access.log -t txt > access-data.txt
 log_sense -f rails -i production.log -t html -o performance.txt
 #+end_example
 
+
 * Change Log
 
 See the [[file:CHANGELOG.org][CHANGELOG]] file.
@@ -109,8 +123,8 @@ Concerning the outputs:
 - HTML reports use [[https://get.foundation/][Zurb Foundation]], [[https://www.datatables.net/][Data Tables]], and [[https://vega.github.io/vega-lite/][Vega Light]], which
   are all downloaded from a CDN
 - The textual format is compatible with [[https://orgmode.org/][Org Mode]] and can be further
-  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to (including HTML
-  and PDF)
+  processed to any format [[https://orgmode.org/][Org Mode]] can be exported to, including HTML
+  and PDF, with the word of warning in the section above. 
 
 * Author and Contributors
 
@@ -118,8 +132,8 @@ Concerning the outputs:
 
 * Known Bugs
 
-No known bugs; an unknown number of unknown bugs.
-(See the open issues for the known bugs.)
+No known bugs; an unknown number of unknown bugs.  (See the open issues for
+the known bugs.)
 
 * License
 

data/Rakefile

@@ -9,7 +9,21 @@ end
 require_relative './lib/log_sense/ip_locator.rb'
 
 desc "Convert Geolocation DB to sqlite"
-task :dbip_to_sqlite3, [:filename] do |tasks, args|
-  filename = args[:filename]
-  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+task :dbip_to_sqlite3, [:year_month] do |tasks, args|
+  filename = "./ip_locations/dbip-country-lite-#{args[:year_month]}.csv"
+
+  if !File.exist? filename
+    puts "Error. Could not find: #{filename}"
+    puts
+    puts 'I see the following files:'
+    puts Dir.glob("ip_locations/dbip-country-lite*").map { |x| "- #{x}\n" }
+    puts ''
+    puts '1. Download (if necessary) a more recent version from: https://db-ip.com/db/download/ip-to-country-lite'
+    puts '2. Save downloaded file to ip_locations/'
+    puts '3. Relaunch with YYYY-MM'
+
+    exit
+  else
+    LogSense::IpLocator::dbip_to_sqlite filename
+  end
 end

data/exe/log_sense

@@ -7,21 +7,15 @@ require 'log_sense.rb'
 #
 
 # this better be here... OptionsParser consumes ARGV
-@command_line = ARGV.join(" ")
-
+@command_line = ARGV.join(' ')
 @options     = LogSense::OptionsParser.parse ARGV
-@input_file  = @options[:input_file] || ARGV[0]
 @output_file = @options[:output_file]
 
-if not @input_file
-  puts "Error: no input file specified."
-  exit
-end
-
-if not File.exist? @input_file
-  puts "Error: input file '#{@input_file}' does not exist"
+if ARGV.map { |x| File.exist?(x) }.include?(false)
+  $stderr.puts "Error: input file(s) '#{ARGV.reject { |x| File.exist(x) }.join(', ')}' do not exist"
   exit 1
 end
+@input_files = ARGV.empty? ? [$stdin] : ARGV.map { |x| File.open(x, 'r') }
 
 #
 # Parse Log and Track Statistics
@@ -36,32 +30,46 @@ when 'apache'
 when 'rails'
   parser_klass = LogSense::RailsLogParser
   cruncher_klass = LogSense::RailsDataCruncher
+else
+  $stderr.puts "Error: input format #{@options[:input_format]} not understood."
+  exit 1
 end
 
-@db = parser_klass.parse @input_file
+$stderr.puts "Parsing input files..." if @options[:verbose]
+@db = parser_klass.parse @input_files
 
-if @options[:output_format]  == "sqlite"
-  ddb = SQLite3::Database.new(@output_file || "db.sqlite3")
+if @options[:output_format] == 'sqlite'
+  $stderr.puts "Saving to SQLite3..." if @options[:verbose]
+  ddb = SQLite3::Database.new(@output_file || 'db.sqlite3')
   b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
   b.step(-1) #=> DONE
   b.finish
 else
+  $stderr.puts "Aggregating data..." if @options[:verbose]
   @data = cruncher_klass.crunch @db, @options
+
+  $stderr.puts "Geolocating..." if @options[:verbose]
   @data = LogSense::IpLocator.geolocate @data
 
+  $stderr.puts "Grouping by country..." if @options[:verbose]
+  country_col = @data[:ips][0].size - 1
+  @data[:countries] = @data[:ips].group_by { |x| x[country_col] }
+
   @ended_at = Time.now
   @duration = @ended_at - @started_at
 
   @data = @data.merge({
                         command: @command_line,
-                        log_file: @input_file,
+                        filenames: ARGV,
+                        log_files: @input_files,
                         started_at: @started_at,
                         ended_at: @ended_at,
-                        duration: @duration
+                        duration: @duration,
+                        width: @options[:width]
                       })
-
   #
   # Emit Output
   #
+  $stderr.puts "Emitting..." if @options[:verbose]
   puts LogSense::Emitter.emit @data, @options
 end

data/lib/log_sense/apache_data_cruncher.rb

@@ -6,7 +6,7 @@ module LogSense
     # @ variables are automatically put in the returned data
     #
 
-    def self.crunch db, options = { limit: 30 }
+    def self.crunch db, options = { limit: 900 }
       first_day_s = db.execute "SELECT datetime from LogLine order by datetime limit 1"
       last_day_s  = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
 
@@ -15,17 +15,17 @@ module LogSense
       @last_day =  last_day_s&.first&.first ? Date.parse(last_day_s[0][0]) : nil
 
       @total_days = 0
-      if @first_day and @last_day
-        @total_days = (@last_day - @first_day).to_i
-      end
+      @total_days = (@last_day - @first_day).to_i if @first_day && @last_day
+
+      @source_files   = db.execute 'SELECT distinct(source_file) from LogLine'
 
-      @log_size       = db.execute "SELECT count(datetime) from LogLine"
+      @log_size       = db.execute 'SELECT count(datetime) from LogLine'
       @log_size       = @log_size[0][0]
 
       @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
       @selfpolls_size = @selfpolls_size[0][0]
 
-      @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
+      @crawlers_size  = db.execute 'SELECT count(datetime) from LogLine where bot == 1'
       @crawlers_size = @crawlers_size[0][0]
 
       @first_day_requested = options[:from_date]
@@ -35,7 +35,7 @@ module LogSense
       @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
 
       @total_days_in_analysis = 0
-      if @first_day_in_analysis and @last_day_in_analysis
+      if @first_day_in_analysis && @last_day_in_analysis
         @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
       end
 
@@ -45,24 +45,24 @@ module LogSense
       filter = [
         (options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
         (options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
-        (options[:only_crawlers] ? "bot == 1" : nil),
-        (options[:ignore_crawlers] ? "bot == 0" : nil),
+        (options[:only_crawlers] ? 'bot == 1' : nil),
+        (options[:ignore_crawlers] ? 'bot == 0' : nil),
         (options[:no_selfpolls] ? "ip != '::1'" : nil),
-        "true"
+        'true'
       ].compact.join " and "
 
       mega = 1024 * 1024
       giga = mega * 1024
       tera = giga * 1024
-      
+ 
       # in alternative to sum(size)
       human_readable_size = <<-EOS
-      CASE 
+      CASE
       WHEN sum(size) <  1024 THEN sum(size) || ' B' 
       WHEN sum(size) >= 1024 AND sum(size) < (#{mega}) THEN ROUND((CAST(sum(size) AS REAL) / 1024), 2) || ' KB' 
       WHEN sum(size) >= (#{mega})  AND sum(size) < (#{giga}) THEN ROUND((CAST(sum(size) AS REAL) / (#{mega})), 2) || ' MB' 
       WHEN sum(size) >= (#{giga}) AND sum(size) < (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{giga})), 2) || ' GB' 
-      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB' 
+      WHEN sum(size) >= (#{tera}) THEN ROUND((CAST(sum(size) AS REAL) / (#{tera})), 2) || ' TB'
       END AS size
       EOS
 
@@ -89,16 +89,18 @@ module LogSense
 
       @daily_distribution = db.execute "SELECT date(datetime), #{human_readable_day}, count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
       @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"
-      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and extension == '.html' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
-      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
-      @reasonable_requests_exts = [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].map {  |x|
-        "extension != '#{x}'"
-      }.join " and "
+      good_statuses = "(status like '2%' or status like '3%')"
+      bad_statuses = "(status like '4%' or status like '5%')"
+      html_page = "(extension like '.htm%')"
+      non_html_page = "(extension not like '.htm%')"
+
+      @most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size}, status from LogLine where #{good_statuses} and #{non_html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+
+      @missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{html_page} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
+      @missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), status from LogLine where #{bad_statuses} and #{filter} group by path order by count(path) desc limit #{options[:limit]}"
 
-      @attacks = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{filter} and (#{@reasonable_requests_exts}) group by path order by count(path) desc  limit #{options[:limit]}"
       @statuses = db.execute "SELECT status, count(status) from LogLine where #{filter} group by status order by status"
 
       @by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{filter} group by date(datetime)"
@@ -115,20 +117,19 @@ module LogSense
       
       @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @streaks = db.execute 'SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime'
       data = {}
 
-      self.instance_variables.each do |variable|
-        var_as_symbol = variable.to_s[1..-1].to_sym
-        data[var_as_symbol] = eval(variable.to_s)
+      instance_variables.each do |variable|
+        var_as_symbol = variable.to_s[1..].to_sym
+        data[var_as_symbol] = instance_variable_get(variable)
       end
+
       data
     end
 
-    private
-
-    def self.date_intersect date1, date2, method
-      if date1 and date2
+    def self.date_intersect(date1, date2, method)
+      if date1 && date2
         [date1, date2].send(method)
       elsif date1
         date1
@@ -138,4 +139,3 @@ module LogSense
     end
   end
 end
-

data/lib/log_sense/apache_log_line_parser.rb

@@ -31,22 +31,21 @@ module LogSense
 
     TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
 
-    HTTP_METHODS=/GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
-    WEBDAV_METHODS=/COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-    OTHER_METHODS=/SEARCH|REPORT/
-    METHOD=/(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-    PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9])/
-    URL=/(?<url>[^ ]+)/
-    REFERER=/(?<referer>[^ ]+)/
-    RETURN_CODE=/(?<status>[1-5][0-9][0-9])/
-    SIZE=/(?<size>[0-9]+|-)/
-
-    USER_AGENT = /(?<user_agent>[^"]+)/
+    HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
+    WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
+    OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
+    METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
+    PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
+    URL = /(?<url>[^ ]+)/
+    REFERER = /(?<referer>[^"]*)/
+    RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
+    SIZE = /(?<size>[0-9]+|-)/
+    USER_AGENT = /(?<user_agent>[^"]*)/
 
     attr_reader :format
 
-    def initialize 
-      @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "#{METHOD} #{URL} #{PROTOCOL}" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
+    def initialize
+      @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
     end
 
     def parse line

data/lib/log_sense/apache_log_parser.rb

@@ -7,10 +7,9 @@ module LogSense
     # parse an Apache log file and return a SQLite3 DB
     #
 
-    def self.parse filename, options = {}
-      content = filename ? File.readlines(filename) : ARGF.readlines
+    def self.parse(streams, options = {})
+      db = SQLite3::Database.new ':memory:'
 
-      db = SQLite3::Database.new ":memory:"
       db.execute "CREATE TABLE IF NOT EXISTS LogLine(
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       datetime TEXT,
@@ -28,15 +27,18 @@ module LogSense
       browser TEXT,
       browser_version TEXT,
       platform TEXT,
-      platform_version TEXT)"
+      platform_version TEXT,
+      source_file TEXT,
+      line_number INTEGER
+      )"
       
-      ins = db.prepare('insert into LogLine (
-                datetime, 
+      ins = db.prepare("insert into LogLine (
+                datetime,
                 ip,
                 user,
                 unique_visitor,
                 method,
-                path, 
+                path,
                 extension,
                 status,
                 size,
@@ -46,44 +48,50 @@ module LogSense
                 browser,
                 browser_version,
                 platform,
-                platform_version)
-              values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+                platform_version,
+                source_file,
+                line_number
+                )
+              values (#{Array.new(18, '?').join(', ')})")
 
       parser = ApacheLogLineParser.new
-      
-      content.each do |line|
-        begin
-          hash = parser.parse line
-          ua = Browser.new(hash[:user_agent], accept_language: "en-us")
-          ins.execute(
-            DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
-            hash[:ip],
-            hash[:userid],
-            unique_visitor_id(hash),
-            hash[:method],
-            hash[:url],
-            (hash[:url] ? File.extname(hash[:url]) : ""),
-            hash[:status],
-            hash[:size].to_i,
-            hash[:referer],
-            hash[:user_agent],
-            ua.bot? ? 1 : 0,
-            (ua.name || ""),
-            (ua.version || ""),
-            (ua.platform.name || ""),
-            (ua.platform.version || "")
-          )
-        rescue StandardError => e
-          STDERR.puts e.message
+
+      streams.each do |stream|
+        stream.readlines.each_with_index do |line, line_number|
+          begin
+            hash = parser.parse line
+            ua = Browser.new(hash[:user_agent], accept_language: 'en-us')
+            ins.execute(
+              DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
+              hash[:ip],
+              hash[:userid],
+              unique_visitor_id(hash),
+              hash[:method],
+              hash[:url],
+              (hash[:url] ? File.extname(hash[:url]) : ''),
+              hash[:status],
+              hash[:size].to_i,
+              hash[:referer],
+              hash[:user_agent],
+              ua.bot? ? 1 : 0,
+              (ua.name || ''),
+              (ua.version || ''),
+              (ua.platform.name || ''),
+              (ua.platform.version || ''),
+              stream == $stdin ? "stdin" : stream.path,
+              line_number
+            )
+          rescue StandardError => e
+            $stderr.puts e.message
+          end
         end
       end
-      
+
       db
     end
 
     def self.unique_visitor_id hash
       "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
     end
-
   end
 end