smart_parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7c862e5efea46fdb350e38f6f9062cc23effb6266168987b8362839245dcc921
+  data.tar.gz: 1df7e38eda4d160d4bd79c48c5540b40540a711eecf12afcf693004991dc9f0a
+SHA512:
+  metadata.gz: bf989652e214df38edcf2e25e6bfe38406471d9f6ca3da2e558d68498c62aa54ca45f5bdbc4623bed99a17a2912bcc544041c223a9e08cf786e8a1df9993495b
+  data.tar.gz: da06b601f51e421d0bd4fe6d43bb4f19010cd7b38d6ca4518eac4e1855d6d02d81553fca433c9a857e329bbfa50b4171584a44aa17469eb12858fae630702bf2

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,12 @@
+AllCops:
+  NewCops: enable
+  Exclude:
+    - bin/**/*
+    - exe/*
+    - Gemfile
+  DisplayCopNames: true
+  TargetRubyVersion: 2.7
+  SuggestExtensions: false
+
+Metrics/BlockLength:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+ruby "~> 2.7.2"
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in smart_parser.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'rspec', '~> 3.0'
+gem 'terminal-table'
+
+group :development do
+  gem 'rubocop'
+end
+
+group :test do
+  gem 'factory_bot'
+  gem 'simplecov'
+end

data/Gemfile.lock

@@ -0,0 +1,83 @@
+PATH
+  remote: .
+  specs:
+    smart_parser (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.0.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    concurrent-ruby (1.1.9)
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    factory_bot (6.2.0)
+      activesupport (>= 5.0.0)
+    i18n (1.10.0)
+      concurrent-ruby (~> 1.0)
+    minitest (5.15.0)
+    parallel (1.21.0)
+    parser (3.1.1.0)
+      ast (~> 2.4.1)
+    rainbow (3.1.1)
+    rake (12.3.3)
+    regexp_parser (2.2.1)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.25.1)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.15.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.16.0)
+      parser (>= 3.1.1.0)
+    ruby-progressbar (1.11.0)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  factory_bot
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  rubocop
+  simplecov
+  smart_parser!
+  terminal-table
+
+RUBY VERSION
+   ruby 2.7.2p137
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Joffily Ferreira
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,107 @@
+# SmartParser
+
+This project aims to provide a _smart_ and easy way to parse any webserver log and print its results in a human-friendly format. It was designed with flexibility in mind. It makes it simple to change parts like Counters and Presenters.
+
+## Design decisions
+
+```ascii
+          ┌────────────────────────────┐
+          │                            ├────┐
+   ┌──────┤          The User          │    │
+   │      │                            │    │
+   │      └─────────────▲──────────────┘    │
+   │                    │                   │
+   │                    │                   │
+   │                    │                   │
+   │         ┌──────────┴─────────┐         │
+   │         │                    │         │
+   │         │                    │         │
+   │      ┌─►│       Parser       │◄─┐      │
+   │      │  │                    │  │      │
+   │      │  │                    │  │      │
+   │      │  └────────────────────┘  │      │
+   │      │                          │      │
+   │      │                          │      │
+   │      │                          │      │
+   │      │                          │      │
+   ▼      │                          │      ▼
+ ┌────────┴───────────┐    ┌─────────┴──────────┐
+ │                    │    │                    │
+ │                    │    │                    │
+ │      Counter       │    │     Presenter      │
+ │                    │    │                    │
+ │                    │    │                    │
+ └────────────────────┘    └────────────────────┘
+
+```
+
+The above diagram shows how the `Parser` can be used by the user. The `Parser` class requires two moving parts: A list of `Counter` and a `Presenter`. By default, the gem has two counters (`MostVisitedCounter` and `UniqVisitorsCounter`), and many presenters see [here](https://github.com/joffilyfe/smart_parser/tree/main/lib/smart_parser/presenters).
+
+This design allows the user to change any required parts and create their parts. It's easier to create a new Counter or even a new Presenter, just following the signature for them.
+
+The default counters use a `Hash` as a container for the parsed lines, it allows us to access the paths with an O(1) access time. Also, the output of counters returns the sorted array with results, it uses the Ruby Quicksort algorithm which in the worst case will do the sort with O(n^2) but on the avg, it sorts using the O(n log n) [not too bad].
+
+Using the `Presenter` not coupled with the `Parser` or even the `Counter` allows us to implement new parsers easily, it should not take more than five minutes.
+
+## Installation
+
+To use it in your application you should add this line to your Gemfile:
+
+```ruby
+gem 'smart_parser'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or, you can install it by yourself running this command:
+
+```bash
+$ gem install smart_parser
+```
+
+## Usage
+
+You have two ways to use this gem:
+1. As binary directly in your $PATH
+2. As library loaded into your project
+
+### Using as binary
+
+Consider that you already installed the gem using the `gem install` method and installed it on your `lib` of the Ruby.
+
+So, you can just run the following command:
+
+```shell
+$ sparser webserver.log
+```
+
+It includes a binary that has all Counters and Presenters and also includes additional options, to check run:
+
+```shell
+$ sparser -h
+```
+
+*Note*: You should take a look at the variety of `Presenters`. Try the `JSONL` for instance.
+
+### Using as library
+
+This is the second way to use the gem. You can just require (`require 'smart_parser'`) and starts using the classes like:
+
+```ruby
+parser = SmartParser::Parser.new(lines: ['/home 127.0.01'])
+```
+
+To have more details about this use, please check the code of the binary included in the gem.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/joffilyfe/smart_parser.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/exe/sparser

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'optparse'
+
+if $PROGRAM_NAME == __FILE__
+  require_relative '../lib/smart_parser'
+else
+  require 'smart_parser'
+end
+require 'benchmark'
+
+USAGE = "Usage: sparser webserver.log [options]"
+
+def parse_options
+  counters = {
+    mostvisited: SmartParser::Counters::MostVisitedCounter,
+    uniqvisitors: SmartParser::Counters::UniqVisitorsCounter
+  }
+  presenters = {
+    table: SmartParser::Presenters::TablePresenter,
+    line: SmartParser::Presenters::LinePresenter,
+    jsonl: SmartParser::Presenters::JSONLPresenter,
+    none: nil
+  }
+  options = {
+    counters: [SmartParser::Counters::MostVisitedCounter, SmartParser::Counters::UniqVisitorsCounter],
+    presenter: SmartParser::Presenters::TablePresenter,
+    benchmark: false
+  }
+
+  OptionParser.new do |opts|
+    opts.banner = USAGE
+    opts.on('-c', '--use c1,c2', Array, "Select available counters: (#{counters.keys.join(', ')})") do |selected|
+      unless selected.empty?
+        options[:counters] = selected.select { |c| counters.keys.include?(c.to_sym) }
+        options[:counters] = options[:counters].map { |c| counters[c.to_sym] }.flatten
+      end
+    end
+    opts.on('-p PRESENTER', '--presenter', presenters,
+            "Select the output presenter: (#{presenters.keys.join(', ')})") do |presenter|
+      options[:presenter] = presenter
+    end
+    opts.on('-b', '--benchmark', 'Print the benchmark for the script') do |bench|
+      options[:benchmark] = bench
+    end
+  end.parse!
+
+  options
+end
+
+def main
+  options = parse_options
+  logfile = File.open(ARGV[0] || abort(USAGE))
+
+  parser = SmartParser::Parser.new(
+    lines: logfile.readlines,
+    counters: options[:counters].map(&:new),
+    presenter: options[:presenter]
+  )
+
+  time = Benchmark.measure do
+    parser.parse!
+  end
+
+  parser.present
+  puts time if options[:benchmark]
+rescue Errno::ENOENT
+  abort("File '#{ARGV[0]}' not found, please check if the path exists.")
+rescue Errno::EISDIR
+  abort("The path '#{ARGV[0]}' is a directory, please use files only.")
+rescue OptionParser::InvalidArgument, OptionParser::MissingArgument => e
+  abort(e.message)
+end
+
+main

data/lib/smart_parser/counters/counter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SmartParser
+  module Counters
+    # This is a super class which difines what other counters
+    # must implements. Every counter which inherance from this
+    # class will use the Hash as data container to store information
+    # about parsed lines.
+    #
+    # Every counter must implements the `to_a` method and it should
+    # return an array ordered from the most relevant to the least relevant.
+    #
+    class Counter
+      def initialize(container: {}, line_regex: SmartParser::LINE_REGEX)
+        @container = container
+        @line_regex = line_regex
+      end
+
+      def count(line)
+        raise NotImplementedError
+      end
+
+      # Returns an Array containing Arrays with two positions inside
+      def to_a
+        raise NotImplementedError
+      end
+
+      # Returns the description for the Counter
+      #
+      # @returns String
+      def description
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/smart_parser/counters/most_visited_counter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'counter'
+
+module SmartParser
+  module Counters
+    # Count the most visited path
+    class MostVisitedCounter < Counter
+      # Parse the received line using the @line_regex and counts
+      # how many times the parsed `path` was received.
+      #
+      # @returns true - when the line was correctly parsed and counted
+      # @returns false - when the line does not matches with the line regex
+      def count(line)
+        match = @line_regex.match(line.strip)
+
+        return false if match.nil?
+
+        path = match[:path]
+
+        if @container[path].nil?
+          @container[path] = 1
+        else
+          @container[path] += 1
+        end
+
+        true
+      end
+
+      # Returns an Array ordered by the most accessed path to the least accessed one
+      #
+      # @returns Array
+      def to_a
+        @container.each.sort_by { |_k, v| v }.reverse.to_a
+      end
+
+      def description
+        'Most visited'
+      end
+    end
+  end
+end

data/lib/smart_parser/counters/uniq_visitors_counter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'set'
+require_relative 'counter'
+
+module SmartParser
+  module Counters
+    # Count the most visited path considering only uniq visitors
+    class UniqVisitorsCounter < Counter
+      # Parse the received line using the @line_regex and counts
+      # how many times the parsed `path` was received considering only unique
+      # visits.
+      #
+      # @returns true - when the line was correctly parsed and counted
+      # @returns false - when the line does not matches with the line regex
+      def count(line)
+        match = @line_regex.match(line.strip)
+
+        return false if match.nil?
+
+        path = match[:path]
+
+        @container[path] ||= Set.new
+        @container[path] << match[:address]
+
+        true
+      end
+
+      # Returns an Array ordered by the most accessed path to the least accessed one
+      #
+      # @returns Array
+      def to_a
+        @container.each.sort_by { |_k, v| v.size }.map { |k, v| [k, v.size] }.reverse
+      end
+
+      def description
+        'Unique visitors'
+      end
+    end
+  end
+end

data/lib/smart_parser/presenters/jsonl_presenter.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SmartParser
+  module Presenters
+    # The JSONLPresenter implements a way to represent the output of the
+    # Counter.to_a as JSONL.
+    class JSONLPresenter
+      class << self
+        def present(rows, description)
+          output = String.new
+
+          rows.each do |line|
+            json = JSON.dump({ path: line[0], visits: line[1], description: description })
+            output << "#{json}\n"
+          end
+
+          output
+        end
+      end
+    end
+  end
+end

data/lib/smart_parser/presenters/line_presenter.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SmartParser
+  module Presenters
+    # The LinePresenter implements a way to represent the output of the
+    # Counter.to_a as a simple lines showing the number of visits.
+    # Every path + visits will be treat as a line for the output.
+    class LinePresenter
+      class << self
+        def present(rows, description)
+          output = String.new("#{description}\n")
+
+          rows.each do |line|
+            output << "#{line[0]} #{line[1]} visits\n"
+          end
+
+          output
+        end
+      end
+    end
+  end
+end

data/lib/smart_parser/presenters/table_presenter.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'terminal-table'
+
+module SmartParser
+  module Presenters
+    # The TablePresenter implements a way to represent the output of the
+    # Counter.to_a using a Table compatible with terminals.
+    class TablePresenter
+      class << self
+        def present(rows, description)
+          Terminal::Table.new(title: description, headings: %w[Path Count], rows: rows).to_s
+        end
+      end
+    end
+  end
+end

data/lib/smart_parser/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SmartParser
+  VERSION = '0.0.1'
+end

data/lib/smart_parser.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative 'smart_parser/version'
+require_relative 'smart_parser/counters/counter'
+require_relative 'smart_parser/counters/most_visited_counter'
+require_relative 'smart_parser/counters/uniq_visitors_counter'
+require_relative 'smart_parser/presenters/line_presenter'
+require_relative 'smart_parser/presenters/table_presenter'
+require_relative 'smart_parser/presenters/jsonl_presenter'
+
+module SmartParser
+  # Regex to search for data separeted by a space
+  LINE_REGEX = /(?<path>.*)\s+(?<address>.*)$/.freeze
+
+  # By defaul the MostVisitedCounter and UniqVisitorsCounter
+  # will be used to parse lines.
+  DEFAULT_COUNTERS = [
+    Counters::MostVisitedCounter,
+    Counters::UniqVisitorsCounter
+  ].freeze
+
+  # By default the LinePresenter will be used to display
+  # the results of the Parser.
+  DEFAULT_PRESENTER = Presenters::LinePresenter
+
+  # The Parser class join every piece of the gem and expose the
+  # results of the Counters and Presenters selected during its
+  # initialization.
+  #
+  # During the construction of a new Parse instance you must
+  # inject the lines that you want to parse. Also, you can select
+  # your own Counter and your own Presenter.
+  #
+  # ==== Attributes
+  # +lines+ - An Array containing strings to be parsed
+  # +counters+ - An Array containing zero or many Counter's.
+  # +presenter+ - A Presenter which will displays the parser result.
+  #
+  # ==== Examples
+  # # Creates a new Parser with only one Counter and with JSONLPresenter
+  # p = Parser.new(
+  #      lines: ['/a 127.8.8.1'],
+  #      counters: [SmartParser::Counters::UniqVisitorsCounter]
+  #      presenter: SmartParser::Presenters::JSONLPresenter
+  # )
+  # # Now you should parse! and show your results
+  # p.parse!
+  # p.present
+  #
+  class Parser
+    attr_reader :lines
+    attr_accessor :presenter
+
+    def initialize(lines: [], counters: DEFAULT_COUNTERS, presenter: DEFAULT_PRESENTER)
+      @lines = lines
+      @counters = counters
+      @presenter = presenter
+      @parsed = false
+    end
+
+    # Verify all @lines and uses the defined counters to generate
+    # metrics about the log lines. It runs only once.
+    # If lines were parsed successfully it will returns true
+    # otherwise false.
+    #
+    # @returns bool
+    def parse!
+      return false if @counters.empty? || @lines.empty? || parsed?
+
+      @lines.each do |line|
+        @counters.each do |counter|
+          counter.count(line.strip)
+        end
+      end
+
+      parsed!
+
+      true
+    end
+
+    # Returns an Array containing data for every selected Counter.
+    #
+    # returns @array
+    def counters_results
+      @counters.map { |counter| { description: counter.description, data: counter.to_a } }
+    end
+
+    # Shows the parsed result using the selected Presenter.
+    # Note: the result will be displayed using the stdout
+    #
+    # @returns nil
+    def present
+      return if @presenter.nil?
+
+      @counters.each do |counter|
+        puts @presenter.present(counter.to_a, counter.description)
+        puts "\n"
+      end
+    end
+
+    # Checks if the parser already ran.
+    #
+    # @returns bool
+    def parsed?
+      @parsed
+    end
+
+    private
+
+    def parsed!
+      @parsed = true
+    end
+  end
+end

data/smart_parser.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative 'lib/smart_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'smart_parser'
+  spec.version       = SmartParser::VERSION
+  spec.authors       = ['Joffily Ferreira']
+  spec.email         = ['joffilyferreira@gmail.com']
+
+  spec.summary       = 'Inspect yours webserver logs'
+  spec.description   = 'This powerfull webserver log parser will help you count access through your routes'
+  spec.homepage      = 'https://github.com/joffilyfe/smart_parser'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.7.2')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = spec.homepage
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  # spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: smart_parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Joffily Ferreira
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2022-02-26 00:00:00.000000000 Z
+dependencies: []
+description: This powerfull webserver log parser will help you count access through
+  your routes
+email:
+- joffilyferreira@gmail.com
+executables:
+- sparser
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/sparser
+- lib/smart_parser.rb
+- lib/smart_parser/counters/counter.rb
+- lib/smart_parser/counters/most_visited_counter.rb
+- lib/smart_parser/counters/uniq_visitors_counter.rb
+- lib/smart_parser/presenters/jsonl_presenter.rb
+- lib/smart_parser/presenters/line_presenter.rb
+- lib/smart_parser/presenters/table_presenter.rb
+- lib/smart_parser/version.rb
+- smart_parser.gemspec
+homepage: https://github.com/joffilyfe/smart_parser
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/joffilyfe/smart_parser
+  source_code_uri: https://github.com/joffilyfe/smart_parser
+  changelog_uri: https://github.com/joffilyfe/smart_parser
+  rubygems_mfa_required: 'true'
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Inspect yours webserver logs
+test_files: []