log_line_parser 0.1.0

data/lib/log_line_parser.rb

@@ -0,0 +1,232 @@
+#!/usr/bin/env ruby
+
+require "log_line_parser/version"
+require "log_line_parser/line_parser"
+require "log_line_parser/apache"
+require "strscan"
+require "time"
+require "date"
+
+module LogLineParser
+  include LineParser
+  extend LineParser::Helpers
+
+  class MalFormedRecordError < StandardError; end
+
+  module Fields
+    # LogFormat "%h %l %u %t \"%r\" %>s %b" common
+    COMMON = [
+      :remote_host,
+      :remote_logname,
+      :remote_user,
+      :time,
+      :first_line_of_request,
+      :last_request_status,
+      :response_bytes,
+    ].freeze
+
+    # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
+    COMBINED = (COMMON + [:referer, :user_agent]).freeze
+  end
+
+  PREDEFINED_FORMATS = {}
+
+  class LogLineTokenizer < Tokenizer
+    setup(%w([ ] - \\ "), ['\s+']) #"
+  end
+
+  define_nodes(RootNode: [nil, nil, [" "]],
+               BasicFieldNode: [nil, " ", []],
+               TimeNode: ["[", "]", []],
+               StringNode: ['"', '"', []])
+
+  class StringEscapeNode < EscapeNode
+    setup('\\', nil, [], ['\\', '"', 't', 'n', 'r'])
+    ESCAPED = {
+      '\\' => '\\',
+      '"' => '"',
+      't' => "\t",
+      'n' => "\n",
+      'r' => "\r",
+    }
+
+    def to_s
+      ESCAPED[@subnodes[0]] || ''.freeze
+    end
+  end
+
+  define_node_nesting(RootNode => [TimeNode, StringNode],
+                      StringNode => [StringEscapeNode])
+
+  class LogLineNodeStack < NodeStack
+    setup(RootNode, BasicFieldNode)
+
+    def to_a
+      root.subnodes.map {|node| node.to_s }
+    end
+
+    def to_hash(record_type=CombinedLogParser)
+      record_type.to_hash(to_a)
+    end
+
+    def to_record(record_type=CombinedLogParser)
+      record_type.create(to_a)
+    end
+  end
+
+  module ClassMethods
+    DATE_TIME_SEP = /:/
+
+    attr_accessor :parse_time_value, :format_strings
+
+    def setup(field_names, format_strings=nil)
+      @field_names = field_names
+      @format_strings = format_strings
+      @number_of_fields = field_names.length
+      @referer_defined = field_names.include?(:referer)
+      @parse_time_value = false
+    end
+
+    def parse(line)
+      fields = LogLineParser.parse(line).to_a
+      unless fields.length == @number_of_fields
+        raise MalFormedRecordError, line
+      end
+      create(fields)
+    end
+
+    def create(log_fields)
+      new(*log_fields).tap do |rec|
+        rec.last_request_status = rec.last_request_status.to_i
+        rec.response_bytes = response_size(rec)
+        rec.time = parse_time(rec.time) if @parse_time_value
+        rec.parse_request
+        rec.parse_referer if @referer_defined
+      end
+    end
+
+    def to_hash(line)
+      values = line.kind_of?(Array) ? line : LogLineParser.parse(line).to_a
+      h = {}
+      @format_strings.each_with_index do |key, i|
+        h[key] = values[i]
+      end
+      parse_request(h)
+      h
+    end
+
+    def parse_request(h)
+      if first_line_of_request = h["%r".freeze]
+        request = first_line_of_request.split(/ /)
+        h["%m"] ||= request.shift
+        h["%H"] ||= request.pop
+        h["%U%q"] ||= request.size == 1 ? request[0] : request.join(" ".freeze)
+      end
+    end
+
+    private
+
+    def response_size(rec)
+      size_str = rec.response_bytes
+      size_str == "-".freeze ? 0 : size_str.to_i
+    end
+
+    def parse_time(time_str)
+      Time.parse(time_str.sub(DATE_TIME_SEP, " ".freeze))
+    end
+  end
+
+  module InstanceMethods
+    SPACE_RE = / /
+    SLASH_RE = /\//
+    SLASH = '/'.freeze
+    SCHEMES =%w(http: https:)
+
+    attr_reader :method, :protocol, :resource
+    attr_reader :referer_scheme, :referer_host, :referer_resource
+
+    def date(offset=0)
+      DateTime.parse((self.time + offset * 86400).to_s)
+    end
+
+    def parse_request
+      request = self.first_line_of_request.split(SPACE_RE)
+      @method = request.shift
+      @protocol = request.pop
+      @resource = request.size == 1 ? request[0] : request.join(" ".freeze)
+    end
+
+    def parse_referer
+      return if self.referer == "-".freeze
+      parts = self.referer.split(SLASH_RE, 4)
+      if SCHEMES.include? parts[0]
+        @referer_scheme = parts[0]
+        @referer_host = parts[2]
+        @referer_resource = parts[3] ? SLASH + parts[3] : SLASH
+      else
+        @referer_scheme = "".freeze
+        @referer_host = "".freeze
+        @referer_resource = self.referer
+      end
+    end
+
+    def referred_from_host?(host_name)
+      @referer_host == host_name
+    end
+  end
+
+  def self.create_record_type(field_names, format_strings)
+    record_type = Struct.new(*field_names)
+    record_type.extend(ClassMethods)
+    record_type.include(InstanceMethods)
+    record_type.setup(field_names, format_strings)
+    record_type
+  end
+
+  def self.parser(log_format)
+    if log_format.kind_of? String
+      format_strings = Apache.parse_log_format(log_format)
+      field_names = Apache.format_strings_to_symbols(format_strings)
+    else
+      format_strings = nil
+      field_names = log_format
+    end
+
+    create_record_type(field_names, format_strings)
+  end
+
+  def self.parse(line)
+    stack = LogLineNodeStack.new
+    tokens = LogLineTokenizer.tokenize(line.chomp)
+    tokens.each {|token| stack.push token }
+    stack
+    # I'm not checking the reason yet, but the following way of pushing
+    # tokens directly into the stack is very slow.
+    #
+    # LogLineTokenizer.tokenize(line.chomp, stack)
+  end
+
+  def self.to_array(line)
+    parse(line).to_a
+  end
+
+  CommonLogParser = parser(Apache::LogFormat::COMMON)
+  CommonLogWithVHParser = parser(Apache::LogFormat::COMMON_WITH_VH)
+  CombinedLogParser = parser(Apache::LogFormat::COMBINED)
+
+  PREDEFINED_FORMATS['common'] = CommonLogParser
+  PREDEFINED_FORMATS['common_with_vh'] = CommonLogWithVHParser
+  PREDEFINED_FORMATS['combined'] = CombinedLogParser
+
+  def self.each_record(record_type: CommonLogParser,
+                       input: ARGF,
+                       error_output: STDERR)
+    input.each_line do |line|
+      begin
+        yield line, record_type.parse(line)
+      rescue MalFormedRecordError => e
+        error_output.print e.message
+      end
+    end
+  end
+end

data/log_line_parser.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'log_line_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "log_line_parser"
+  spec.version       = LogLineParser::VERSION
+  spec.required_ruby_version = ">= 2.0.0"
+  spec.authors       = ["HASHIMOTO, Naoki"]
+  spec.email         = ["hashimoto.naoki@gmail.com"]
+
+  spec.summary       = %q{A simple parser of Apache access logs}
+  spec.description   = %q{A simple parser of Apache access logs: it parses a line of Apache access log and turns it into an array of strings or a Hash object. And from the command line, you can use it as a conversion tool of file formats or as a filtering tool of access records.}
+  spec.homepage      = "https://github.com/nico-hn/LogLineParser"
+  spec.license       = "MIT"
+
+  #  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  #  # delete this section to allow pushing this gem to any host.
+  #  if spec.respond_to?(:metadata)
+  #    spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  #  else
+  #    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  #  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/samples/output/access-to-two-specific-files.log

@@ -0,0 +1,2 @@
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /end.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"

data/samples/output/all-but-bots-and-not-found.log

@@ -0,0 +1,10 @@
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /end.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir HTTP/1.1" 301 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html?q=try+to+find+something HTTP/1.1" 200 432 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external2.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"

data/samples/output/all-records-related-to-subdir_index.log

@@ -0,0 +1,4 @@
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html?q=try+to+find+something HTTP/1.1" 200 432 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"

data/samples/output/index-page-accessed-by-bot.log

@@ -0,0 +1 @@
+192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

data/samples/output/referred-from-external-site.log

@@ -0,0 +1 @@
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"

data/samples/sample_combined_log.log

@@ -0,0 +1,12 @@
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /end.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /non-existent.html HTTP/1.1" 404 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir HTTP/1.1" 301 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html?q=try+to+find+something HTTP/1.1" 200 432 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external2.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
+192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

data/samples/sample_config.yml

@@ -0,0 +1,46 @@
+---
+host_name: www.example.org
+resources:
+ - /subdir/index.html
+match:
+ - :access_to_under_resources?
+ - :referred_from_resources?
+match_type: any
+output_log_name: all-records-related-to-subdir_index
+---
+host_name: www.example.org
+resources:
+ - /end.html
+ - /subdir/big.pdf
+match:
+ - :access_to_resources?
+match_type: any
+output_log_name: access-to-two-specific-files
+---
+host_name: www.example.org
+resources:
+ - /
+match:
+ - :access_to_under_resources?
+match_type: any
+ignore_match:
+ - :access_by_bots?
+ - :not_found?
+output_log_name: all-but-bots-and-not-found
+---
+host_name: www.example.org
+resources:
+ - /index.html
+match:
+ - :access_to_resources?
+ - :access_by_bots?
+match_type: all
+output_log_name: index-page-accessed-by-bot
+---
+host_name: www.example.net
+resources:
+ - /external.html
+match:
+ - :referred_from_resources?
+match_type: all
+output_log_name: referred-from-external-site

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: log_line_parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- HASHIMOTO, Naoki
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 'A simple parser of Apache access logs: it parses a line of Apache access
+  log and turns it into an array of strings or a Hash object. And from the command
+  line, you can use it as a conversion tool of file formats or as a filtering tool
+  of access records.'
+email:
+- hashimoto.naoki@gmail.com
+executables:
+- log_line_parser
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/log_line_parser
+- lib/log_line_parser.rb
+- lib/log_line_parser/apache.rb
+- lib/log_line_parser/command_line_interface.rb
+- lib/log_line_parser/line_parser.rb
+- lib/log_line_parser/moe.rb
+- lib/log_line_parser/query.rb
+- lib/log_line_parser/utils.rb
+- lib/log_line_parser/version.rb
+- log_line_parser.gemspec
+- samples/output/access-to-two-specific-files.log
+- samples/output/all-but-bots-and-not-found.log
+- samples/output/all-records-related-to-subdir_index.log
+- samples/output/index-page-accessed-by-bot.log
+- samples/output/referred-from-external-site.log
+- samples/sample_combined_log.log
+- samples/sample_config.yml
+homepage: https://github.com/nico-hn/LogLineParser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.3
+signing_key: 
+specification_version: 4
+summary: A simple parser of Apache access logs
+test_files: []