weblog-parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 213d9f51148d7c7c5201648b8409c9996c8fa6250bc7256766029e2973989c9a
+  data.tar.gz: fb7e70f03efcf6d711133731ee759702a3639dba7d6117eaff28afd8e7c06d20
+SHA512:
+  metadata.gz: dcd77fe9ae16abbea86aa980d93bc98a6ada1bd9999e84752a45057dfc0eb5da7e9355e224e7ff07c58d3cfb0bff1a0cea62b7aa7c210b851067b739ab7ac56c
+  data.tar.gz: 542e1293a56caeceff20eca7487998a973ac3f69b13dae306197479fb725c907f9bc5b1c85797b7467340dc1a910ab6dcd8702b72dbf4c5d6f66b78c8299f63b

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.5
+before_install: gem install bundler -v 1.17.2

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at davidmorton0@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in LogParser.gemspec
+gemspec
+
+gem "byebug", "~> 11.1"

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    LogParser (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (11.1.1)
+    minitest (5.14.0)
+    rake (10.5.0)
+
+PLATFORMS
+  x64-mingw32
+
+DEPENDENCIES
+  LogParser!
+  bundler
+  byebug (~> 11.1)
+  minitest (~> 5.0)
+  rake (~> 10.0)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 David Morton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,197 @@
+# Webserver Log Parser - Readme
+
+## Installing
+
+  `gem install weblog-parser`
+
+## Usage
+
+wlparser reads a webserver logfile and counts page visits and unique page views.
+It uses a command-line interface.
+
+  `wlparser -h`
+  `wlparser --help`
+
+Shows a list of options
+
+  `wlparser -f logfile.log`
+  `wlparser --file logfile.log`
+
+Reads a log file and display results:
+
+  `wlparser -m 'logfile1.log logfile2.log'`
+  `wlparser --multiple_files 'logfile1.log logfile2.log'`
+
+Reads a list of log files in quotes and displays results.  All files give
+using -f or -m options will be read and the output combined.
+
+If no files are specified, the default file 'webserver.log' will be read.
+
+  `wlparser -c`
+  `wlparser --color`
+
+Displays colored text output.  Colors can be change in Constants.rb.
+
+  `wlparser -C`
+  `wlparser --no_color`
+
+Disables colored text output.
+
+  `wlparser -v`
+  `wlparser --verbose`
+
+Shows extra information, including all validation warnings.
+
+  `wlparser -q`
+  `wlparser --quiet`
+
+Displays minimal information i.e. only important warnings.  Will still write
+information to a file if this option is selected.  Disables verbose.
+
+  `wlparser -o`
+  `wlparser --output_file info.txt`
+
+Writes output to file.  Default is 'log_info.txt' if no file chosen, although
+this will only work if this is the last argument given.
+
+  `wlparser -t`
+  `wlparser --timestamp`
+
+Adds a timestamp to the output file.  If an output file is given that already
+exists, this is turned on automatically.
+
+  `wlparser -x`
+  `wlparser --text`
+
+Sets file output format to text, similar to that displayed (default).
+
+ `wlparser -j`
+ `wlparser --json`
+
+Sets file output format to json.
+
+ `wlparser -4`
+ `wlparser --ip4_validation`
+
+Validates ip addresses using ip4 format (default).
+
+ `wlparser -6`
+ `wlparser --ip6_validation`
+
+Validates ip addresses using ip6 format.
+
+ `wlparser -6`
+ `wlparser --ip4ip6_validation`
+
+Validates ip addresses if it matches either ip4 or ip6 format.
+
+ `wlparser -I`
+ `wlparser --no_ip_validation`
+
+Does not validate ip addresses, assumes they are all valid.
+
+ `wlparser -p`
+ `wlparser --path_validation`
+
+Validates webpage path (default).
+
+ `wlparser -P`
+ `wlparser --no_path_validation`
+
+Does not validate webpage path, assumes they are all valid.
+
+ `wlparser -r`
+ `wlparser --remove_invalid`
+
+Ignore logs in files if either ip address or path is invalid.
+
+ `wlparser -R`
+ `wlparser --warn_invalid`
+
+Warns about logs with invalid ip addresss or path, but still reads them
+(default)
+
+  `wlparser -g`
+  `wlparser --page_visits`
+
+Displays page visits in results and in text file output (default).
+
+  `wlparser -g`
+  `wlparser --page_visits`
+
+Does not display page visits in results or text file output.
+
+  `wlparser -u`
+  `wlparser --unique_page_views`
+
+Displays unique page views in results and in text file output (default).
+
+  `wlparser -U`
+  `wlparser --no_unique_page_views`
+
+Displays page visits in results and in text file output (default).
+
+##Log Format
+
+Logs should be on separate lines.
+There should be a space separator between the webpage path and the ip address.
+
+Example log with ip4 address:
+  `\webpage\index 123.123.123.123`
+
+Logs can use either using ip4 addresses or ip6 addresses.  
+
+ip4 addresses should be valid i.e. between 0.0.0.0 and 255.255.255.255, although
+you can skip this check.
+
+Example log with ip6 address:
+  `\webpage\index 1234:1234:1234:1234:1234:1234:1234:1234`
+
+ip6 addresses can be compressed e.g.
+
+  `\webpage\index 1234:1234::1234`
+
+## Testing
+
+Tests can be run using:
+
+  `rake test`
+
+Tests have been separated into
+* unit tests - test methods in each class
+* integration tests - test the whole app
+* performance - parses a log file with 10,000 logs and a log with 100 logs
+100 times.  Calculates the time taken and logs parsed/second.  The log files
+are a mixture of ip4 and ip6 addresses.
+
+## App structure
+
+A class diagram can be found here: https://tinyurl.com/tky2f74
+Note that the dependencies to Constants are not shown.
+
+## Executables
+
+* wlparser - Starts the app
+
+### Classes
+
+* Parser - Holds the log information and changes the format
+* LogReader - Loads files then reads logs.  Validates logs, ip addresses and paths
+* ipValidator - Validates ip addresses
+* PathValidator - Validates the path for the webpage
+* OptionHandler - Sets the options from the command line arguments given
+* Formatter - Formats information for text or display output
+* OutputProcessor - Assembles the information for output
+* WarningHandler - Handles the warnings found when parsing the logs
+
+### Modules
+
+* LogParser - Calls the methods in order
+* Constants - Contains default options and other constants used in the app
+* TestData - Contains the data used in the tests
+* ColorText - Adds color to text
+* Version - Gives the current version number
+
+### Logs
+
+* test_logs contains log files used in testing

data/Rakefile

@@ -0,0 +1,12 @@
+ENV['APP_ENV'] = 'test'
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+ENV['RACK_ENV'] = 'test'
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "LogParser"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/wlparser

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path('../../lib', __dir__)
+require 'log_parser'
+
+LogParser.parse

data/lib/log_parser.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+Dir.glob(File.join(__dir__, 'log_parser', '*.rb'))
+   .sort
+   .each { |file| require file }
+
+# main file for app
+module LogParser
+  def self.parse
+    @options = OptionHandler.new.options
+
+    log_reader = LogReader.new(
+      options: { file_list: @options[:file_list],
+                 path_validation: @options[:path_validation],
+                 ip_validation: @options[:ip_validation],
+                 log_remove: @options[:log_remove] }
+    ).load_logs
+
+    parser = Parser.new(log_reader: log_reader,
+                        quiet: @options[:quiet],
+                        verbose: @options[:verbose])
+    parser.count_views
+
+    output_processor = OutputProcessor.new(parser: parser, options: @options)
+    puts output_processor.output_to_display
+
+    return unless @options[:output_file]
+
+    output_processor.write_to_file(format: @options[:output_format])
+  end
+end

data/lib/log_parser/color_text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Changes text color
+module ColorText
+  COLOR_CODE = { black: 30,
+                 red: 31,
+                 green: 32,
+                 yellow: 33,
+                 blue: 34,
+                 magenta: 35,
+                 cyan: 36,
+                 gray: 37,
+                 white: 38 }.freeze
+
+  def colorize(text, color)
+    "\e[#{COLOR_CODE[color]}m#{text}\e[0m"
+  end
+
+  def colorize_if(text, color, change_color = false)
+    change_color ? colorize(text, color) : text
+  end
+end

data/lib/log_parser/constants.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# Constants file
+module Constants
+  # Options that can be changed ------------------------------------------------
+  DEFAULT_OPTIONS = { ip_validation: :ip4,
+                      path_validation: true,
+                      page_visits: true,
+                      unique_page_views: true,
+                      output_format: :text,
+                      file_list: [] }
+
+  DEFAULT_LOG = 'webserver.log'
+
+  OUTPUT_COLORS = { title: :cyan,        # can be :black, :red, :green, :yellow,
+                    line_break: :yellow, # :blue, :magenta, :cyan, :gray, :white
+                    columns: %i[gray green red], # if more columns,
+                    log: :cyan,          # final color will be used
+                    options: :magenta }.freeze
+
+  WARNING_COLORS = { true => :red,       # important warnings color
+                     false => :yellow,   # non-important warnings color
+                     :none => :green }.freeze
+  # ---------------------------------------------------------------------------
+
+  OPTION_DESCRIPTIONS = {
+    verbose: ->(value) { "verbose: #{(value || false)}" },
+    quiet: ->(value) { "quiet: #{(value || false)}" },
+    highlighting: ->(value) { "highlighting: #{(value || false)}" },
+    file_list: ->(files) { "file list: #{files ? files.join(', ') : '- '}" },
+    output_file: ->(output_file) { "output file: #{output_file || '- '}" },
+    timestamp: ->(value) { "timestamp: #{(value || false)}" },
+    output_format: ->(format) { "output format: #{format}" },
+    ip_validation: lambda { |validation|
+      "ip validation: #{VALIDATION_NAMES[validation]}"
+    },
+    log_remove: ->(value) { "ignore invalid logs: #{(value || false)}" },
+    path_validation: ->(value) { "path validation: #{(value || false)}" },
+    page_visits: ->(value) { "show page visits: #{(value || false)}" },
+    unique_page_views: lambda { |value|
+      "show unique page views: #{(value || false)}"
+    }
+  }.freeze
+
+  VALIDATION_NAMES = { log: 'log',
+                       ip4: 'ip4 address',
+                       ip6: 'ip6 address',
+                       ip4_ip6: 'ip4/ip6 address',
+                       none: 'none',
+                       path: 'path' }.freeze
+
+  LOG_WARNINGS = { file: { name: 'File Error', important: true },
+                   log: { name: 'Log Format Error', important: true },
+                   ip4: { name: 'Ip4 Address Format Error', important: false },
+                   ip6: { name: 'Ip6 Address Format Error', important: false },
+                   ip4_ip6: { name: 'Ip Address Format Error',
+                              important: false },
+                   path: { name: 'Path Format Error',
+                           important: false } }.freeze
+
+  WARNINGS_JSON = { file: :fileError,
+                    log: :logFormatError,
+                    ip4: :ip4AddressFormatError,
+                    ip6: :ip6AddressFormatError,
+                    ip4_ip6: :ipAddressFormatError,
+                    path: :pathFormatError }.freeze
+
+  INFO_TITLES = { visits: 'Page Visits',
+                  unique_views: 'Unique Page Views' }.freeze
+
+  DESCRIPTORS = { visits: ['', 'visit'],
+                  unique_views: ['', 'unique view'] }.freeze
+
+  VALID_ADDRESS = {
+    none: proc { true },
+    ip4: proc { |address| address.match(VALID_IP4) },
+    ip6: proc { |address| address.match(VALID_IP6) },
+    ip4_ip6: proc { |address|
+      address.match(VALID_IP4) || address.match(VALID_IP6)
+    }
+  }.freeze
+
+  VALID_LOG = '\S+\s([a-zA-Z0-9:.]+)'
+
+  VALID_PATH = '^(/[a-zA-Z0-9.$_+\\-!*(),\']+/?)*/?$'
+
+  VALID_IP4 = '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2' \
+              '[0-4][0-9]|[01]?[0-9][0-9]?)$'
+
+  VALID_IP6 = '^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:)'\
+              '{1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]'\
+              '{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}'\
+              '(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]'\
+              '{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|'\
+              '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]'\
+              '{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::'\
+              '(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}'\
+              '[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-f'\
+              'A-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.)'\
+              '{3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$'
+end

data/lib/log_parser/formatter.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+# Formats text for output
+class Formatter
+  include ColorText
+  include Constants
+
+  attr_accessor :output
+
+  def initialize
+    @output = []
+  end
+
+  def add_title(title:, add_color: false)
+    output.push(colorize_if(title, OUTPUT_COLORS[:title], add_color))
+    line_break = colorize_if(('-' * title.length),
+                             OUTPUT_COLORS[:line_break], add_color)
+    output.unshift(line_break)
+    output.push(line_break)
+  end
+
+  def add_row(row:, descriptor:, add_color: false)
+    output.push(row.map.with_index { |item, i|
+      colorize_if(add_descriptor(item, descriptor[i]),
+                  OUTPUT_COLORS[:columns][i], add_color)
+    }.join(' '))
+  end
+
+  def add_descriptor(item, descriptor)
+    if item.is_a?(Integer) && (descriptor != '')
+      format('%<item>s %<descriptor>s', item: item,
+                                        descriptor: pluralise(descriptor, item))
+    else
+      item
+    end
+  end
+
+  def format_info(view_info:, add_color: false)
+    @output = []
+    add_title(title: view_info[:title], add_color: add_color)
+    view_info[:info].sort_by { |page, views| [-views, page] }
+                    .each do |row|
+                      add_row(row: row,
+                              descriptor: view_info[:descriptor],
+                              add_color: add_color)
+                    end
+    @output
+  end
+
+  def format_log_info(log_info:, add_color: false)
+    files = log_info[:files_read].map { |file| File.absolute_path(file) }
+                                 .join(",\n")
+    format("\n%<file_info>s\n%<logs_read>s\n%<logs_added>s",
+           file_info: colorize_if("Files read: #{files}",
+                                  OUTPUT_COLORS[:log], add_color),
+           logs_read: colorize_if("Logs read: #{log_info[:logs_read]}",
+                                  OUTPUT_COLORS[:log], add_color),
+           logs_added: colorize_if("Logs added: #{log_info[:logs_added]}",
+                                   OUTPUT_COLORS[:log], add_color))
+  end
+
+  def format_minimal_warnings(warnings:, add_color: false)
+    warnings_list = warnings.filter { |_type, info|
+      info[:important] && !info[:warnings].empty?
+    }.map { |_type, info|
+      color = WARNING_COLORS[true]
+      text = [warning_summary(info[:name], info[:warnings].length),
+              info[:warnings]].join("\n")
+      colorize_if(text, color, add_color)
+    }
+    warnings_list.flatten.join("\n")
+  end
+
+  def format_full_warnings(warnings:, add_color: false)
+    warnings_list = warnings.map do |_type, info|
+      if info[:warnings].empty?
+        color = WARNING_COLORS[:none]
+        text = [warning_summary(info[:name], info[:warnings].length)]
+      else
+        color = WARNING_COLORS[info[:important]]
+        text = [warning_summary(info[:name], info[:warnings].length),
+                info[:warnings]]
+      end
+      colorize_if(text.join("\n"), color, add_color)
+    end
+    warnings_list.flatten.join("\n")
+  end
+
+  def format_normal_warnings(warnings:, add_color: false)
+    warnings_list = warnings.map do |_type, info|
+      if !info[:warnings].empty? && info[:important]
+        color = WARNING_COLORS[true]
+        text = [warning_summary(info[:name], info[:warnings].length),
+                info[:warnings]].join("\n")
+      elsif info[:important]
+        color = WARNING_COLORS[:none]
+        text = [warning_summary(info[:name], info[:warnings].length)].join("\n")
+      else
+        color_key = info[:warnings].empty? ? :none : false
+        color = WARNING_COLORS[color_key]
+        text = [warning_summary(info[:name], info[:warnings].length)].join("\n")
+      end
+      colorize_if(text, color, add_color)
+    end
+    warnings_list.flatten.join("\n")
+  end
+
+  def pluralise(item, number)
+    item + (number != 1 ? 's' : '')
+  end
+
+  def warning_summary(name, number)
+    format('%<name>ss: %<number>d %<warnings>s',
+           name: name, number: number, warnings: pluralise('warning', number))
+  end
+
+  def format_options(options:, add_color: false)
+    options = OPTION_DESCRIPTIONS.map do |k, _v|
+      OPTION_DESCRIPTIONS[k].call(options[k])
+    end
+    text = format("\nOptions selected:\n\n%<options>s\n",
+                  options: options.join(",\n"))
+    colorize_if(text, OUTPUT_COLORS[:options], add_color)
+  end
+end

data/lib/log_parser/ip_validator.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Validates ip addresses
+class IpValidator
+  include Constants
+
+  attr_reader :ip_address, :validation
+
+  def initialize(ip_address:, validation:)
+    @ip_address = ip_address
+    @validation = VALID_ADDRESS[validation || :none]
+  end
+
+  def valid?
+    @validation.call(ip_address)
+  end
+end

data/lib/log_parser/log_reader.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+# Reads logs
+class LogReader
+  include Constants
+
+  attr_accessor :logs_read, :logs_added
+  attr_reader :read_log, :warnings, :files_read, :options
+
+  def initialize(options: {})
+    @read_log = Hash.new { |h, k| h[k] = [] }
+    @options = options
+    @warnings = []
+    @logs_read = 0
+    @logs_added = 0
+    @files_read = []
+    @options[:file_list] = [DEFAULT_LOG] if options[:file_list] == []
+  end
+
+  def load_logs
+    options[:file_list].each { |file| load_log(file: file) }
+    self
+  end
+
+  def load_log(file:)
+    begin
+      File.open(file, 'r').each.with_index do |line, i|
+        add_log(log: line, line_number: i + 1, file: file)
+      end
+      @files_read.push(file)
+    rescue
+      warnings.push(type: :file,
+                    message: format(' - File not found: %<file>s', file: file))
+    end
+    self
+  end
+
+  def add_log(log:, line_number: 1, file:)
+    @logs_read += 1
+
+    log_valid = valid_log?(log: log)
+    add_warning_if(type: :log, line_number: line_number, file: file,
+                   add_if: !log_valid)
+    return self unless log_valid
+
+    path, ip_address = log.split(' ')
+    ip_valid = valid_ip?(ip_address: ip_address)
+    path_valid = !options[:path_validation] || valid_path?(path: path)
+
+    add_warning_if(type: options[:ip_validation], line_number: line_number,
+                   file: file, add_if: !ip_valid)
+    add_warning_if(type: :path, line_number: line_number, file: file,
+                   add_if: !path_valid)
+
+    if (ip_valid && path_valid) || !options[:log_remove]
+      @logs_added += 1
+      read_log[path].push(ip_address)
+    end
+    self
+  end
+
+  def add_warning_if(type:, line_number:, file:, add_if:)
+    return unless add_if
+
+    warnings.push(type: type,
+                  message: log_warning_message(
+                    name: VALIDATION_NAMES[type],
+                    line_number: line_number,
+                    file: file
+                  ))
+  end
+
+  def log_warning_message(name:, line_number:, file:)
+    format(' - Invalid %<name>s%<line_file>s',
+           name: name,
+           line_file: add_line_and_file_name(line_number, file))
+  end
+
+  def add_line_and_file_name(line_number, file)
+    format('%<file>s - line %<line>d', file: add_file_name(file),
+                                       line: line_number)
+  end
+
+  def add_file_name(file)
+    format(' - File: %<file>s', file: file)
+  end
+
+  def valid_ip?(ip_address:)
+    IpValidator.new(ip_address: ip_address,
+                    validation: options[:ip_validation]).valid?
+  end
+
+  def valid_log?(log:)
+    log.match(VALID_LOG)
+  end
+
+  def valid_path?(path:)
+    PathValidator.new(path: path).valid?
+  end
+end

data/lib/log_parser/option_handler.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+# Parses command line options
+class OptionHandler
+  include Constants
+
+  attr_reader :options
+
+  def initialize
+    @options = DEFAULT_OPTIONS.clone
+    @options[:file_list] = []
+
+    begin
+      OptionParser.new do |opts|
+        opts.on('-v', '--verbose', 'Show extra information') do
+          options[:verbose] = true unless options[:quiet]
+        end
+
+        opts.on('-q', '--quiet',
+                'No display except important warnings.  Disables verbose') do
+          options[:quiet] = true
+          options[:verbose] = false
+        end
+
+        opts.on('-c', '--color', 'Enables colored display text') do
+          options[:highlighting] = true
+        end
+
+        opts.on('-C', '--no_color',
+                'Disables colored display text (default)') do
+          options[:highlighting] = false
+        end
+
+        opts.on('-f', '--file [FILE]',
+                'Log file to read. Default is webserver.log') do |file|
+          if file
+            options[:file_list].push(file)
+          else
+            unless ENV['APP_ENV'] == 'test'
+              puts 'Missing input file name.  Exiting.'
+            end
+            exit 50
+          end
+        end
+
+        opts.on('-m', "--multiple_files ['FILE_LIST']",
+                'Read a list of log files in quotes') do |file_list|
+          if file_list
+            options[:file_list] += file_list.split(' ')
+          else
+            unless ENV['APP_ENV'] == 'test'
+              puts 'Missing input file list.  Exiting.'
+            end
+            exit 50
+          end
+        end
+
+        opts.on('-o', '--output_file [FILE]', 'Write output to file') do |file|
+          options[:output_file] = file || 'log_info.txt'
+        end
+
+        opts.on('-t', '--timestamp', 'Add timestamp to output file') do
+          options[:timestamp] = true
+        end
+
+        opts.on('-x', '--text', 'Sets file output format to text (default)') do
+          options[:output_format] = :text
+        end
+
+        opts.on('-j', '--json', 'Sets file output format to json') do
+          options[:output_format] = :json
+        end
+
+        opts.on('-h', '--help', 'Shows help') do
+          puts opts if ENV['APP_ENV'] != 'test'
+          exit 51
+        end
+
+        opts.on('-4', '--ip4_validation',
+                'Validate ip addresses using ip4 format (default)') do
+          options[:ip_validation] = :ip4
+        end
+
+        opts.on('-6', '--ip6_validation',
+                'Validate ip addresses using ip6 format') do
+          options[:ip_validation] = :ip6
+        end
+
+        opts.on('-i', '--ip4ip6_validation',
+                'Validate ip addresses using either ip4 or ip6 format') do
+          options[:ip_validation] = :ip4_ip6
+        end
+
+        opts.on('-I', '--no_ip_validation',
+                'No validatation of ip addresses') do
+          options[:ip_validation] = :none
+        end
+
+        opts.on('-r', '--remove_invalid',
+                'Ignore log if invalid ip addresss or path') do
+          options[:log_remove] = true
+        end
+
+        opts.on('-R', '--warn_invalid',
+                'Warn but not ignore log if invalid ip address or path
+                (default)') do
+          options[:log_remove] = false
+        end
+
+        opts.on('-p', '--path_validation', 'Validate webpage path (default)') do
+          options[:path_validation] = true
+        end
+
+        opts.on('-P', '--no_path_validation',
+                'Does not validate webpage path') do
+          options[:path_validation] = false
+        end
+
+        opts.on('-g', '--page_visits', 'Show page visits (default)') do
+          options[:page_visits] = true
+        end
+
+        opts.on('-G', '--no_page_visits', 'Do not show page visits') do
+          options[:page_visits] = false
+        end
+
+        opts.on('-u', '--unique_page_views',
+                'Show unique page views (default)') do
+          options[:unique_page_views] = true
+        end
+
+        opts.on('-U', '--no_unique_page_views',
+                'Do not show unique page views') do
+          options[:unique_page_views] = false
+        end
+      end.parse!
+    rescue OptionParser::InvalidOption => e
+      puts e if ENV['APP_ENV'] != 'test'
+      exit
+    end
+  end
+end

data/lib/log_parser/output_processor.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'json'
+# Processes output for display or write to file
+class OutputProcessor
+  include Constants
+
+  attr_reader :add_color, :parser
+
+  def initialize(parser:, options:)
+    @parser = parser
+    @options = options
+    @add_color = options ? @options[:highlighting] : false
+  end
+
+  def name_output_file
+    if File.exist?(@options[:output_file]) || @options[:timestamp]
+      timestamp_filename(@options[:output_file])
+    else
+      @options[:output_file]
+    end
+  end
+
+  def timestamp_filename(file)
+    dir  = File.dirname(file)
+    base = File.basename(file, '.*')
+    time = Time.now.strftime('%d-%m-%y_%H-%M-%S')
+    ext  = File.extname(file)
+    File.join(dir, "#{base}_#{time}#{ext}")
+  end
+
+  def output_to_display
+    output = []
+    if @options[:quiet]
+      output.push(parser.formatted_minimal_warnings(add_color: add_color))
+    else
+      if @options[:page_visits]
+        output.push(parser.formatted_page_views(view_type: :visits,
+                                                add_color: add_color))
+      end
+      if @options[:unique_page_views]
+        output.push(parser.formatted_page_views(view_type: :unique_views,
+                                                add_color: add_color))
+      end
+      output.push(parser.formatted_log_info(add_color: add_color))
+      if @options[:verbose]
+        output.unshift(Formatter.new.format_options(options: @options,
+                                                    add_color: add_color))
+        output.push('', parser.formatted_full_warnings(add_color: add_color))
+      else
+        output.push('', parser.formatted_normal_warnings(add_color: add_color))
+      end
+    end
+    output.join("\n")
+  end
+
+  def output_to_file_text
+    output = []
+    if @options[:page_visits]
+      output.push(parser.formatted_page_views(view_type: :visits).join("\n"))
+    end
+    if @options[:unique_page_views]
+      output.push(parser.formatted_page_views(
+        view_type: :unique_views
+      ).join("\n"))
+    end
+    output.push(parser.formatted_log_info)
+    if @options[:verbose]
+      output.push(parser.formatted_full_warnings)
+    else
+      output.push(parser.formatted_normal_warnings)
+    end
+    output.join("\n")
+  end
+
+  def output_to_file_json
+    JSON.pretty_generate parser.hash_format(verbose: @options[:verbose])
+  end
+
+  def write_to_file(format:)
+    file = name_output_file
+    f = File.new(file, 'w')
+    format_select = { text: -> { output_to_file_text },
+                      json: -> { output_to_file_json } }
+    f.write format_select[format].call
+    f.close
+
+    puts format('Output written to: %<file>s', file: file)
+  end
+end

data/lib/log_parser/parser.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# Parses log information
+class Parser
+  include Constants
+
+  attr_reader :page_views, :formatter, :log_reader
+
+  def initialize(log_reader: {}, quiet: false, verbose: false)
+    @page_views = {}
+    @log_reader = log_reader
+    @formatter = Formatter.new
+    @warning_handler = nil
+  end
+
+  def warnings
+    WarningHandler.new(warnings: log_info[:warnings])
+                  .store_warning_info(warning_info: LOG_WARNINGS)
+                  .warnings_summary
+  end
+
+  def count_views(logs: log_reader.read_log)
+    logs.each do |page, ip_addresses|
+      @page_views[page] = { visits: ip_addresses.length,
+                            unique_views: ip_addresses.uniq.length }
+    end
+  end
+
+  def view_info(view_type:)
+    { title: INFO_TITLES[view_type],
+      descriptor: DESCRIPTORS[view_type],
+      info: @page_views.map { |page, views| [page, views[view_type]] } }
+  end
+
+  def log_info
+    { files_read: (log_reader == {} ? [] : log_reader.files_read),
+      logs_read: (log_reader == {} ? 0 : log_reader.logs_read),
+      logs_added: (log_reader == {} ? 0 : log_reader.logs_added),
+      warnings: (log_reader == {} ? [] :  log_reader.warnings) }
+  end
+
+  def formatted_log_info(add_color: false)
+    formatter.format_log_info(log_info: log_info, add_color: add_color)
+  end
+
+  def formatted_page_views(view_type:, add_color: false)
+    formatter.format_info(view_info: view_info(view_type: view_type),
+                          add_color: add_color)
+  end
+
+  def formatted_full_warnings(add_color: false)
+    formatter.format_full_warnings(warnings: warnings, add_color: add_color)
+  end
+
+  def formatted_minimal_warnings(add_color: false)
+    formatter.format_minimal_warnings(warnings: warnings, add_color: add_color)
+  end
+
+  def formatted_normal_warnings(add_color: false)
+    formatter.format_normal_warnings(warnings: warnings, add_color: add_color)
+  end
+
+  def hash_format(verbose:)
+    output = {}
+    output['filesRead'] = log_info[:files_read]
+    output['logsRead'] = log_info[:logs_read]
+    output['logsAdded'] = log_info[:logs_added]
+    output['pageVisits'] = {}
+    output['uniquePageViews'] = {}
+    page_views.each do |page, views|
+      output['pageVisits'][page] = views[:visits]
+      output['uniquePageViews'][page] = views[:unique_views]
+    end
+    if verbose
+      warning_summary = warnings.map do |type, info|
+        { WARNINGS_JSON[type] => {
+          'numberWarnings': info[:warnings].length,
+          'warnings': info[:warnings]
+        } }
+      end
+    else
+      warning_summary = warnings.map do |type, info|
+        if info[:important]
+          { WARNINGS_JSON[type] => {
+            'numberWarnings': info[:warnings].length,
+            'messages': info[:warnings]
+          } }
+        else
+          { WARNINGS_JSON[type] => {
+            'numberWarnings': info[:warnings].length
+          } }
+        end
+      end
+    end
+    output['warnings'] = warning_summary
+    output
+  end
+end

data/lib/log_parser/path_validator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# Vaidates the webpath (slug)
+class PathValidator
+  include Constants
+
+  attr_reader :path
+
+  def initialize(path:)
+    @path = path
+  end
+
+  def valid?
+    path.match(VALID_PATH)
+  end
+end

data/lib/log_parser/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Stores version number
+module LogParser
+  VERSION = '0.1.0'
+end

data/lib/log_parser/warning_handler.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# Handles warnings found when reading logs
+class WarningHandler
+  include Constants
+
+  attr_reader :warnings, :warning_info
+
+  def initialize(warnings: [])
+    @warnings = warnings
+  end
+
+  def store_warning_info(warning_info: {})
+    @warning_info = warning_info
+    self
+  end
+
+  def warnings_summary
+    summary = {}
+    warning_info.each do |type, info|
+      summary[type] = {}
+      summary[type][:name] = info[:name]
+      summary[type][:important] = info[:important]
+      summary[type][:warnings] = @warnings
+                                 .filter { |warning| warning[:type] == type }
+                                 .map { |warning| warning[:message] }
+    end
+    summary
+  end
+end

data/weblog_parser.gemspec

@@ -0,0 +1,40 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "log_parser/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "weblog-parser"
+  spec.version       = LogParser::VERSION
+  spec.authors       = ["David Morton"]
+  spec.email         = ["davidmorton0@gmail.com"]
+
+  spec.summary       = %q{A command line parser for web server logs}
+  spec.homepage      = "https://github.com/davidmorton0/LogParser"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    #spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+    #spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = "https://github.com/davidmorton0/LogParser"
+    #spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = ["wlparser"]
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.1.4"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: weblog-parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- David Morton
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.4
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- davidmorton0@gmail.com
+executables:
+- wlparser
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- bin/wlparser
+- lib/log_parser.rb
+- lib/log_parser/color_text.rb
+- lib/log_parser/constants.rb
+- lib/log_parser/formatter.rb
+- lib/log_parser/ip_validator.rb
+- lib/log_parser/log_reader.rb
+- lib/log_parser/option_handler.rb
+- lib/log_parser/output_processor.rb
+- lib/log_parser/parser.rb
+- lib/log_parser/path_validator.rb
+- lib/log_parser/version.rb
+- lib/log_parser/warning_handler.rb
+- weblog_parser.gemspec
+homepage: https://github.com/davidmorton0/LogParser
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/davidmorton0/LogParser
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A command line parser for web server logs
+test_files: []