http-log-analyzer 0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5b4140aef82a3b635f50830132f5725b21a2f753d17f78cab80ef918807d3410
+  data.tar.gz: 91d4884ba4b346211cfdb5707aca91229f5df5dea3c8e6e96657e500b2025ee5
+SHA512:
+  metadata.gz: e3ed61db57fb6c32a382695c4d518429e936972c3ec954e0199d3eba54199ec6c47d1b226335a2e3aad59242f790c19cfa8746f0436ea5fce46b57585ba6878f
+  data.tar.gz: 8df22a7d0d7344c119caf3cbf05f7d3a678f794094713223179dd7e4634f4d75532303d623d1d44fff1fd05c7a038e0deed99e868fbe91f2e33e7b4ba2590569

data/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+*.gem

data/Rakefile

@@ -0,0 +1,3 @@
+require 'rubygems/tasks'
+
+Gem::Tasks.new

data/TODO.txt

@@ -0,0 +1,32 @@
+collapse referers by domain of referer
+
+ensure domain is not in referers list
+
+remove 'android-app://com.google.android.gm' from referers list
+  or count under Google?
+
+cache log files (as Marshalled data) to reduce parsing time
+
+move ignore-data to external files
+
+use external list for user-agent strings
+  http://www.useragentstring.com/pages/useragentstring.php?name=All
+  https://www.webmasterworld.com/search_engine_spiders/4849056.htm
+
+detect users & sessions
+
+store parsed data
+  store cache in files
+
+generate monthly reports
+
+detect users & sessions
+  first parse, then process
+
+add configuration file (in Ruby)
+  site/report name
+  domain names
+  ignored IPs
+
+use more recent geo-IP databases?
+  https://dev.maxmind.com/geoip/geoip2/downloadable/

data/bin/http-log-analyzer

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'http-log-analyzer'
+require 'simple_option_parser'
+
+options = SimpleOptionParser.parse(ARGV,
+  domain: nil,
+  ignored_cities: nil,
+  ignored_browsers: nil,
+  ignored_referers: nil,
+  ignored_paths: nil,
+  period: nil,
+)
+
+%i{ignored_cities ignored_browsers ignored_referers ignored_paths}.each do |key|
+  if (file = options[key])
+    options[key] = HTTPLogAnalyzer::Importer.parse_list(File.read(file))
+  end
+end
+
+importer = HTTPLogAnalyzer::Importer.new(options)
+
+last_filename = nil
+file_size = nil
+ARGF.each_line do |line|
+  if ARGF.filename != last_filename
+    last_filename = ARGF.filename
+    if ARGF.filename == '-'
+      file_size = 1
+    else
+      file_size = File.size(ARGF.filename)
+    end
+  end
+  STDERR.print "%-20s %6d %3d%%\r" % [
+    ARGF.filename,
+    ARGF.lineno,
+    ARGF.filename == '-' ? 100 : ((ARGF.pos.to_f / file_size) * 100).ceil,
+  ]
+  begin
+    importer.process_line(ARGF.filename, ARGF.lineno, line)
+  rescue HTTPLogAnalyzer::ParseError => e
+    warn "#{e}: #{ARGF.filename}:#{ARGF.lineno}: #{line}"
+  end
+end
+STDERR.puts ' ' * ENV['COLUMNS'].to_i
+
+importer.report

data/http-log-analyzer.gemspec

@@ -0,0 +1,28 @@
+require_relative 'lib/http-log-analyzer/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'http-log-analyzer'
+  s.version       = HTTPLogAnalyzer::VERSION
+  s.author        = 'John Labovitz'
+  s.email         = 'johnl@johnlabovitz.com'
+
+  s.summary       = %q{Analyze HTTP log files.}
+  # s.description   = %q{}
+  s.homepage      = 'http://github.com/jslabovitz/http-log-analyzer'
+  s.license       = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_path  = 'lib'
+
+  s.add_dependency 'addressable', '~> 2.5'
+  s.add_dependency 'geoip', '~> 1.6'
+  s.add_dependency 'http-log-parser', '~> 0'
+  s.add_dependency 'mime-types', '~> 3.1'
+  s.add_dependency 'simple_option_parser', '~> 0.3'
+  s.add_dependency 'user_agent_parser', '~> 2.4'
+
+  s.add_development_dependency 'rake', '~> 12.3'
+  s.add_development_dependency 'rubygems-tasks', '~> 0.2'
+end

data/lib/http-log-analyzer.rb

@@ -0,0 +1,24 @@
+require 'net/http/status'
+require 'date'
+
+require 'http_log_parser'
+require 'user_agent_parser'
+require 'addressable/uri'
+require 'mime/types'
+require 'geoip'
+
+module HTTPLogAnalyzer
+
+  class ParseError < Exception; end
+
+end
+
+require 'http-log-analyzer/entry'
+require 'http-log-analyzer/importer'
+require 'http-log-analyzer/stats'
+require 'http-log-analyzer/element'
+require 'http-log-analyzer/element/referer'
+require 'http-log-analyzer/element/request'
+require 'http-log-analyzer/element/source'
+require 'http-log-analyzer/element/status'
+require 'http-log-analyzer/element/user_agent'

data/lib/http-log-analyzer/element.rb

@@ -0,0 +1,19 @@
+module HTTPLogAnalyzer
+
+  class Element
+
+    def self.parse(string)
+      $cache ||= {}
+      $cache[self] ||= {}
+      $cache[self][string] ||= new.tap { |e| e.parse(string) }
+    end
+
+    def normalize_uri!(uri)
+      uri.normalize!
+      uri.scheme = 'http' if uri.scheme == 'https'
+      uri.host = uri.host.downcase.sub(/^(www|m)\./, '').sub(/\.$/, '') if uri.host
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/element/referer.rb

@@ -0,0 +1,39 @@
+module HTTPLogAnalyzer
+
+  class Element
+
+    class Referer < Element
+
+      QueryKeys = %w{q p searchfor wd}
+
+      attr_accessor :uri
+      attr_accessor :query
+
+      def parse(string)
+        if string != '-'
+          @uri = Addressable::URI.parse(string) or raise ParseError, "Can't parse URI: #{string}"
+          normalize_uri!(@uri)
+          if @uri.host
+            # normalize Facebook link-shims
+            if @uri.host =~ /^(l|lm)\.facebook\.com$/ || @uri.path.start_with?('/l.php')
+              @uri.host = 'facebook.com'
+              @uri.path = '/'
+              @uri.query = nil
+            end
+            if (values = @uri.query_values)
+              QueryKeys.each do |key|
+                if (value = values[key]) && !value.empty?
+                  @query = values[key]
+                  break
+                end
+              end
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/element/request.rb

@@ -0,0 +1,21 @@
+module HTTPLogAnalyzer
+
+  class Element
+
+    class Request < Element
+
+      attr_accessor :uri
+      attr_accessor :mime_types
+
+      def parse(string)
+        @method, uri_string, @version = string.split(/\s+/)
+        @uri = Addressable::URI.parse(uri_string) or raise ParseError, "Can't parse URI: #{uri_string}"
+        normalize_uri!(@uri)
+        @mime_types = MIME::Types.type_for(@uri.path)
+      end
+
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/element/source.rb

@@ -0,0 +1,32 @@
+module HTTPLogAnalyzer
+
+  class Element
+
+    class Source < Element
+
+      attr_accessor :address
+      attr_accessor :country
+      attr_accessor :region
+      attr_accessor :city
+
+      def parse(string)
+        result = $geo_ip.city(string)
+        if result
+          city, region, country = %i{city_name real_region_name country_name}.map do |key|
+            value = result.send(key)
+            value.to_s.empty? ? nil : value
+          end
+          @address = result.ip
+          @country = country
+          @region = [region, country].join(', ') if region
+          @city = [city, region, country].join(', ') if city
+        else
+          @ip = string
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/element/status.rb

@@ -0,0 +1,23 @@
+module HTTPLogAnalyzer
+
+  class Element
+
+    class Status < Element
+
+      attr_accessor :code
+      attr_accessor :message
+
+      def parse(string)
+        @code = string.to_i
+        @message = Net::HTTP::STATUS_CODES[@code]
+      end
+
+      def class
+        @code / 100
+      end
+
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/element/user_agent.rb

@@ -0,0 +1,38 @@
+module HTTPLogAnalyzer
+
+  class Element
+
+    class UserAgent < Element
+
+      attr_accessor :browser
+      attr_accessor :system
+
+      def parse(string)
+        if string != '-'
+          user_agent = $user_agent_parser.parse(string)
+          @browser = case (family = user_agent.family.strip)
+          when nil, 'Other'
+            'other'
+          else
+            family
+          end
+          @system = case (name = user_agent.os.name.strip)
+          when /^Windows\s/
+            'Windows'
+          when /^Mac OS/i
+            'macOS'
+          when 'Other', nil
+            'other'
+          when 'Ubuntu', 'Fedora', 'SUSE'
+            'Linux'
+          else
+            name
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/entry.rb

@@ -0,0 +1,28 @@
+module HTTPLogAnalyzer
+
+  class Entry
+
+    attr_accessor :timestamp
+    attr_accessor :source
+    attr_accessor :request
+    attr_accessor :status
+    attr_accessor :referer
+    attr_accessor :user_agent
+
+    def initialize(info)
+      # 12/Feb/2016:09:59:04 +0000
+      @timestamp = DateTime.strptime(info[:datetime], '%d/%b/%Y:%H:%M:%S %Z')
+      @source = Element::Source.parse(info[:ip])
+      @request = Element::Request.parse(info[:request])
+      @status = Element::Status.parse(info[:status])
+      @referer = Element::Referer.parse(info[:referer])
+      @user_agent = Element::UserAgent.parse(info[:user_agent])
+    end
+
+    def calendar_week
+      '%s - %s' % [1, 7].map { |d| DateTime.commercial(@timestamp.year, @timestamp.cweek, d).to_date }
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/importer.rb

@@ -0,0 +1,90 @@
+module HTTPLogAnalyzer
+
+  class Importer
+
+    StatusKeys = {
+      4 => :client_error_statuses,
+      5 => :server_error_statuses,
+    }
+
+    def initialize(
+      domain:,
+      ignored_ips: nil,
+      ignored_cities: nil,
+      ignored_browsers: nil,
+      ignored_referers: nil,
+      ignored_paths: nil,
+      period: nil
+    )
+      @domain = domain
+      @ignored_ips = ignored_ips || {}
+      @ignored_cities = ignored_cities || {}
+      @ignored_browsers = ignored_browsers || {}
+      @ignored_referers = ignored_referers || {}
+      @ignored_paths = ignored_paths || {}
+      @period = period ? parse_period(period) : nil
+      @log_parser = HttpLogParser.new
+      $user_agent_parser = UserAgentParser::Parser.new
+      $geo_ip = GeoIP.new(File.expand_path('/usr/local/var/GeoIP/GeoLiteCity.dat'))
+      @stats = Stats.new
+    end
+
+    def parse_period(period)
+      Range.new(*period.split(' - ', 2).map { |d| DateTime.parse(d) })
+    end
+
+    def process_line(file, line_num, line)
+      begin
+        data = @log_parser.parse_line(line)
+      rescue => e
+        raise ParseError, "Can't parse line: #{line}"
+      end
+      entry = Entry.new(data)
+      if @period && !@period.cover?(entry.timestamp)
+        # ignore timestamp out of specified period
+      elsif ignore?(entry)
+        @ignored_ips[entry.source.address] = true
+      elsif page?(entry)
+        if (statuses_key = StatusKeys[entry.status.class])
+          @stats.add(statuses_key, "#{entry.status.code}: #{entry.request.uri.path}")
+        end
+        @stats.add(:source_country, entry.source.country)
+        @stats.add(:source_region, entry.source.region)
+        @stats.add(:source_city, entry.source.city)
+        @stats.add(:wv_source_city, entry.source.city) if entry.source.region == 'West Virginia, United States'
+        @stats.add(:pages, entry.request.uri.path)
+        @stats.add(:via, entry.referer.uri) unless entry.referer&.uri&.host == @domain
+        @stats.add(:searches, entry.referer.query) if entry.referer&.query
+        @stats.add(:browsers, entry.user_agent.browser)
+        @stats.add(:systems, entry.user_agent.system)
+        @stats.add(:dates, entry.calendar_week)
+      end
+    end
+
+    def ignore?(entry)
+      @ignored_ips[entry.source.address] ||
+        @ignored_browsers[entry.user_agent.browser] ||
+        @ignored_cities[entry.source.city] ||
+        (entry.referer&.uri && @ignored_referers[entry.referer&.uri&.host]) ||
+        @ignored_paths[entry.request.uri.path]
+    end
+
+    def page?(entry)
+      entry.request.uri.path !~ %r{/.*?https?://} &&    # bad URL construction (from bot)
+        !entry.request.uri.path.empty? &&
+        entry.request.mime_types.empty?  # pages have empty MIME types
+    end
+
+    def report
+      @stats.report
+    end
+
+    def self.parse_list(text)
+      Hash[
+        text.split(/\n/).map { |s| s.sub(/#.*/, '').strip }.reject(&:empty?).map { |s| [s, true] }
+      ]
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/stats.rb

@@ -0,0 +1,79 @@
+module HTTPLogAnalyzer
+
+  class Stats
+
+    Sections = {
+      via: %q{Top 20 sites people came from (aka 'referers')},
+      searches: %q{Searches},
+      source_country: %q{Top 20 countries visitors came from},
+      source_region: %q{Top 20 regions visitors came from},
+      source_city: %q{Top 20 cities visitors came from},
+      wv_source_city: %q{Top 20 WV cities visitors came from},
+      browsers: %q{Top 20 browsers of visitors},
+      systems: %q{Top 20 operating systems of visitors},
+      pages: %q{Top 20 pages visited at our site},
+      informational_statuses: %q{Informational: request received, continuing process},
+      success_statuses: %q{Success: action successfully received, understood, accepted},
+      redirection_statuses: %q{Redirection: further action must be taken in order to complete the request},
+      client_error_statuses: %q{Client error: request contains bad syntax or cannot be fulfilled},
+      server_error_statuses: %q{Server error: server failed to fulfill an apparently valid request},
+      dates: %q{Visitors per week},
+    }
+
+    def initialize
+      @stats = Hash[ Sections.keys.map { |k| [k, {}] } ]
+    end
+
+    def add(key, value)
+      if value
+        table = @stats[key] or raise "No stats for key #{key.inspect}"
+        table[value] ||= 0
+        table[value] += 1
+      end
+    end
+
+    def report
+      @stats.each do |field, counts|
+        next if counts.empty?
+        puts; puts "#{Sections[field]}:"
+        total = counts.values.sum
+        if field == :dates
+          counts = counts.sort_by { |k, v| k }
+        else
+          counts = counts.sort_by { |k, v| v }.reverse[0..19]
+        end
+        counts.each do |label, count|
+          puts '    %s' % BarGraphItem.new(label: label, count: count, total: total)
+        end
+      end
+    end
+
+    class BarGraphItem
+
+      def initialize(label:, count:, total:, width: 50)
+        @label = label
+        @count = count
+        @total = total
+        @width = width
+        @fraction = @count.to_f / @total
+      end
+
+      def bar
+        n = (@fraction * @width).ceil.to_i
+        ('.' * (@width - n)) + ('#' * n)
+      end
+
+      def to_s
+        '%s %3d%% %6d  %s' % [
+          bar,
+          (@fraction * 100).ceil.to_i,
+          @count,
+          @label,
+        ]
+      end
+
+    end
+
+  end
+
+end

data/lib/http-log-analyzer/version.rb

@@ -0,0 +1,5 @@
+module HTTPLogAnalyzer
+
+  VERSION = '0.3'
+
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: http-log-analyzer
+version: !ruby/object:Gem::Version
+  version: '0.3'
+platform: ruby
+authors:
+- John Labovitz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: geoip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: http-log-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: simple_option_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: user_agent_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: 
+email: johnl@johnlabovitz.com
+executables:
+- http-log-analyzer
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Rakefile
+- TODO.txt
+- bin/http-log-analyzer
+- http-log-analyzer.gemspec
+- lib/http-log-analyzer.rb
+- lib/http-log-analyzer/element.rb
+- lib/http-log-analyzer/element/referer.rb
+- lib/http-log-analyzer/element/request.rb
+- lib/http-log-analyzer/element/source.rb
+- lib/http-log-analyzer/element/status.rb
+- lib/http-log-analyzer/element/user_agent.rb
+- lib/http-log-analyzer/entry.rb
+- lib/http-log-analyzer/importer.rb
+- lib/http-log-analyzer/stats.rb
+- lib/http-log-analyzer/version.rb
+homepage: http://github.com/jslabovitz/http-log-analyzer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Analyze HTTP log files.
+test_files: []