log_line_parser 0.1.0

data/lib/log_line_parser/line_parser.rb

@@ -0,0 +1,211 @@
+#!/usr/bin/env ruby
+
+module LineParser
+  class Tokenizer
+    class << self
+      attr_reader :special_token_re, :non_special_token_re
+
+      def tokenize(str, tokens=[])
+        @scanner.string = str
+        token = true # to start looping, you should assign a truthy value
+        while token
+          tokens.push token if token = scan_token
+        end
+
+        tokens.push @scanner.rest unless @scanner.eos?
+        tokens
+      end
+
+      def setup(special_tokens, unescaped_special_tokens=[])
+        @special_tokens = special_tokens
+        @unescaped_special_tokens = unescaped_special_tokens
+        @scanner = StringScanner.new("".freeze)
+        @special_token_re, @non_special_token_re = compose_re(@special_tokens)
+      end
+
+      private
+
+      def scan_token
+        @scanner.scan(@special_token_re) ||
+          @scanner.scan_until(@non_special_token_re)
+      end
+
+      def compose_special_tokens_str(special_tokens)
+        sorted = special_tokens.sort {|x, y| y.length <=> x.length }
+        escaped = sorted.map {|token| Regexp.escape(token) }
+        escaped.concat @unescaped_special_tokens if @unescaped_special_tokens
+        escaped.join('|')
+      end
+
+      def compose_re(special_tokens)
+        tokens_str = compose_special_tokens_str(special_tokens)
+        return Regexp.compile(tokens_str), Regexp.compile("(?=#{tokens_str})")
+      end
+    end
+  end
+
+  class NodeStack
+    attr_reader :current_node, :root
+
+    class << self
+      attr_reader :root_node_class, :default_node_class
+
+      def setup(root_node_class, default_node_class)
+        @root_node_class = root_node_class
+        @default_node_class = default_node_class
+      end
+    end
+
+    def initialize
+      @root = self.class.root_node_class.new
+      @default_node_class = self.class.default_node_class
+      @current_node = @root
+    end
+
+    def push_node(node)
+      @current_node.push node
+      node.node_below = @current_node
+      @current_node = node
+    end
+
+    def pop
+      popped = @current_node
+      @current_node = @current_node.node_below
+      popped.node_below = nil
+      popped
+    end
+
+    def push_token(token)
+      @current_node.push token
+    end
+
+    def push(token)
+      if @current_node.kind_of? EscapeNode
+        push_escaped_token(token)
+      elsif @current_node.end_tag?(token)
+        pop
+      elsif subnode_class = @current_node.subnode_class(token)
+        push_node(subnode_class.new)
+      elsif @current_node.can_ignore?(token)
+        nil
+      else
+        push_node(@default_node_class.new) if @current_node == @root
+        push_token(token)
+      end
+    end
+
+    def push_escaped_token(token)
+      part_to_be_escaped = @current_node.part_to_be_escaped(token)
+      remaining_part = nil
+      if part_to_be_escaped
+        remaining_part = @current_node.remove_escaped_part(token)
+        push_token(part_to_be_escaped)
+      end
+      pop
+      push_token(remaining_part) if remaining_part
+    end
+  end
+
+  class Node
+    attr_accessor :node_below
+
+    class << self
+      attr_reader :start_tag, :end_tag, :subnode_classes
+      attr_reader :start_tag_to_subnode, :tokens_to_be_ignored
+
+      def register_subnode_classes(*subnode_classes)
+        @subnode_classes = subnode_classes
+        subnode_classes.each do |subnode|
+          @start_tag_to_subnode[subnode.start_tag] = subnode
+        end
+      end
+
+      def setup(start_tag, end_tag, to_be_ignored=[])
+        @start_tag_to_subnode = {}
+        @tokens_to_be_ignored = []
+        @start_tag = start_tag
+        @end_tag = end_tag
+        @tokens_to_be_ignored.concat(to_be_ignored) if to_be_ignored
+      end
+    end
+
+    attr_reader :subnodes
+
+    def initialize
+      @subnodes = []
+      @self_class = self.class
+      @cannot_ignore = @self_class.tokens_to_be_ignored.empty?
+    end
+
+    def accept(visitor, memo=nil)
+      visitor.visit(self, memo)
+    end
+
+    def to_s
+      @subnodes.join
+    end
+
+    def subnode_class(token)
+      @self_class.start_tag_to_subnode[token]
+    end
+
+    def end_tag?(token)
+      @self_class.end_tag == token
+    end
+
+    def can_ignore?(token)
+      if @cannot_ignore
+        false
+      else
+        @self_class.tokens_to_be_ignored.include?(token)
+      end
+    end
+
+    def push(token)
+      @subnodes.push token
+    end
+  end
+
+  class EscapeNode < Node
+    class << self
+      attr_reader :to_be_escaped, :to_be_escaped_re
+
+      def setup(start_tag, end_tag, to_be_ignored=[], to_be_escaped=[])
+        super(start_tag, end_tag, to_be_ignored)
+        @to_be_escaped = to_be_escaped
+        @to_be_escaped_re = compile_to_be_escaped_re(to_be_escaped)
+      end
+
+      def compile_to_be_escaped_re(to_be_escaped)
+        re_str = to_be_escaped.map {|e| Regexp.escape(e) }.join("|")
+        /\A(?:#{re_str})/
+      end
+    end
+
+    def remove_escaped_part(token)
+      token.sub(@self_class.to_be_escaped_re, ''.freeze)
+    end
+
+    def part_to_be_escaped(token)
+      @self_class.to_be_escaped.each do |e|
+        return e if token.start_with?(e)
+      end
+      nil
+    end
+  end
+
+  module Helpers
+    def define_nodes(class_name_and_setup_values)
+      class_name_and_setup_values.each do |name, setup_values|
+        new_node_class = const_set(name, Class.new(Node))
+        new_node_class.setup(*setup_values)
+      end
+    end
+
+    def define_node_nesting(parent_children={})
+      parent_children.each do |parent, children|
+        parent.register_subnode_classes(*children)
+      end
+    end
+  end
+end

data/lib/log_line_parser/moe.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'log_line_parser'
+require 'log_line_parser/utils'
+
+# MoeLogParser is added from the personal needs of the original author,
+# and the LogFormat for it is not a widely used format.
+# You may remove this file if you don't need it.
+# (MOE is the acronym of the organization's name for which the author
+# is working at the time of the first release of this program.)
+
+module LogLineParser
+  # CombinedLogFormat + "%D"
+  MoeLogFormat = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D"
+  MoeLogParser = parser(MoeLogFormat)
+  PREDEFINED_FORMATS['moe'] = MoeLogParser
+end
+

data/lib/log_line_parser/query.rb

@@ -0,0 +1,290 @@
+#!/usr/bin/env ruby
+
+module LogLineParser
+  class Query
+    class NotAllowableMethodError < StandardError; end
+
+    module HttpMethods
+      OPTIONS = "OPTIONS"
+      GET = "GET"
+      HEAD = "HEAD"
+      POST = "POST"
+      PUT = "PUT"
+      DELETE = "DELETE"
+      TRACE = "TRACE"
+      CONNECT = "CONNECT"
+      PATCH = "PATCH"
+    end
+
+    TAIL_SLASH_RE = /\/$/
+    SLASH = '/'
+    DEFAULT_BOTS = %w(
+Googlebot
+Googlebot-Mobile
+Mediapartners-Google
+Bingbot
+Slurp
+Baiduspider
+BaiduImagespider
+BaiduMobaider
+YetiBot
+)
+
+    ALLOWABLE_METHODS = [
+      :access_by_bots?,
+      :referred_from_resources?,
+      :referred_from_under_resources?,
+      :access_to_resources?,
+      :access_to_under_resources?,
+      :status_code_206?,
+      :status_code_301?,
+      :status_code_304?,
+      :status_code_404?,
+      :partial_content?,
+      :moved_permanently?,
+      :not_modified?,
+      :not_found?,
+      :options_method?,
+      :get_method?,
+      :head_method?,
+      :post_method?,
+      :put_method?,
+      :delete_method?,
+      :trace_method?,
+      :connect_method?,
+      :patch_method?,
+    ]
+
+    module ConfigFields
+      HOST_NAME = "host_name"
+      RESOURCES = "resources"
+      MATCH = "match"
+      IGNORE_MATCH = "ignore_match"
+      OUTPUT_LOG_NAME = "output_log_name"
+      MATCH_TYPE = "match_type" # The value should be "all" or "any".
+    end
+
+    def self.compile_bots_re(bot_names=DEFAULT_BOTS)
+      bots_str = bot_names.map {|name| Regexp.escape(name) }.join("|")
+      Regexp.compile(bots_str, Regexp::IGNORECASE)
+    end
+
+    DEFAULT_BOTS_RE = compile_bots_re
+
+    def self.access_by_bots?(record, bots_re=DEFAULT_BOTS_RE)
+      bots_re =~ record.user_agent
+    end
+
+    def self.referred_from_resources?(record, resources=[])
+      resources.include?(record.referer_resource)
+    end
+
+    def self.referred_from_under?(record, path)
+        record.referer_resource.start_with?(path)
+    end
+
+    def self.access_to_resources?(record, resources=[])
+      resources.include?(record.resource)
+    end
+
+    def self.access_to_under?(record, path)
+      record.resource.start_with?(path)
+    end
+
+    class << self
+      def register_query_to_log(option, logs)
+        query = Query.new(domain: option[ConfigFields::HOST_NAME],
+                          resources: option[ConfigFields::RESOURCES])
+        queries = option[ConfigFields::MATCH]
+        reject_unacceptable_queries(queries)
+        log = logs[option[ConfigFields::OUTPUT_LOG_NAME]]
+        match_type = option[ConfigFields::MATCH_TYPE]
+        ignore_match = option[ConfigFields::IGNORE_MATCH]
+        reject_unacceptable_queries(ignore_match) if ignore_match
+        compile_query(match_type, log, query, queries, ignore_match)
+      end
+
+      private
+
+      def reject_unacceptable_queries(queries)
+        unacceptable_queries = queries - ALLOWABLE_METHODS
+        unless unacceptable_queries.empty?
+          message = error_message_for_unacceptable_queries(unacceptable_queries)
+          raise NotAllowableMethodError.new(message)
+        end
+      end
+
+      def error_message_for_unacceptable_queries(unacceptable_queries)
+        query_names = unacceptable_queries.join(", ")
+        if unacceptable_queries.length == 1
+          "An unacceptable query is set: #{query_names}"
+        else
+          "Unacceptable queries are set: #{query_names}"
+        end
+      end
+
+      def log_if_all_match(log, query, queries)
+        proc do |line, record|
+          if queries.all? {|method| query.send(method, record) }
+            log.print line
+          end
+        end
+      end
+
+      def log_if_any_match(log, query, queries)
+        proc do |line, record|
+          if queries.any? {|method| query.send(method, record) }
+            log.print line
+          end
+        end
+      end
+
+      def log_if_all_match_but(log, query, queries, ignore_match)
+        proc do |line, record|
+          if queries.all? {|method| query.send(method, record) } and
+              not ignore_match.any? {|method| query.send(method, record) }
+            log.print line
+          end
+        end
+      end
+
+      def log_if_any_match_but(log, query, queries, ignore_match)
+        proc do |line, record|
+          if queries.any? {|method| query.send(method, record) } and
+              not ignore_match.any? {|method| query.send(method, record) }
+            log.print line
+          end
+        end
+      end
+
+      def compile_query(match_type, log, query, queries, ignore_match)
+        if match_type == "all".freeze
+          if ignore_match
+            return log_if_all_match_but(log, query, queries, ignore_match)
+          end
+          log_if_all_match(log, query, queries)
+        else
+          if ignore_match
+            return log_if_any_match_but(log, query, queries, ignore_match)
+          end
+          log_if_any_match(log, query, queries)
+        end
+      end
+    end
+
+    def initialize(domain: nil, resources: [])
+      @domain = domain
+      @resources = normalize_resources(resources)
+      @normalized_resources = normalize_resources(resources)
+      @normalized_dirs = @normalized_resources - @resources
+    end
+
+    def access_by_bots?(record, bots_re=DEFAULT_BOTS_RE)
+      bots_re =~ record.user_agent
+    end
+
+    def referred_from_resources?(record)
+      if_matching_domain(record) and
+        @normalized_resources.include?(record.referer_resource)
+    end
+
+    def referred_from_under_resources?(record)
+      referer_resource = record.referer_resource
+      if_matching_domain(record) and
+        @normalized_dirs.include?(referer_resource) or
+        @resources.any?{|target| referer_resource.start_with?(target) }
+    end
+
+    def access_to_resources?(record)
+      @normalized_resources.include?(record.resource)
+    end
+
+    def access_to_under_resources?(record)
+      resource = record.resource
+      @normalized_dirs.include?(resource) or
+        @resources.any? {|target| resource.start_with?(target) }
+    end
+
+    def status_code_206?(record)
+      record.last_request_status == 206
+    end
+
+    def status_code_301?(record)
+      record.last_request_status == 301
+    end
+
+    def status_code_304?(record)
+      record.last_request_status == 304
+    end
+
+    def status_code_404?(record)
+      record.last_request_status == 404
+    end
+
+    alias :partial_content? :status_code_206?
+    alias :moved_permanently? :status_code_301?
+    alias :not_modified? :status_code_304?
+    alias :not_found? :status_code_404?
+
+    def options_method?(record)
+      record.method == HttpMethods::OPTIONS
+    end
+
+    def get_method?(record)
+      record.method == HttpMethods::GET
+    end
+
+    def head_method?(record)
+      record.method == HttpMethods::HEAD
+    end
+
+    def post_method?(record)
+      record.method == HttpMethods::POST
+    end
+
+    def put_method?(record)
+      record.method == HttpMethods::PUT
+    end
+
+    def delete_method?(record)
+      record.method == HttpMethods::DELETE
+    end
+
+    def trace_method?(record)
+      record.method == HttpMethods::TRACE
+    end
+
+    def connect_method?(record)
+      record.method == HttpMethods::CONNECT
+    end
+
+    def patch_method?(record)
+      record.method == HttpMethods::PATCH
+    end
+
+    private
+
+    def if_matching_domain(record)
+      # When @domain is not set, it should be ignored.
+      not @domain or @domain == record.referer_host
+    end
+
+    def normalize_resources(resources)
+      [].tap do |normalized|
+        resources.each do |resource|
+          # record.referer_resource is expected to return '/'
+          # even when the value of record.referer doesn't end
+          # with a slash (e.g. 'http://www.example.org').
+          # So in the normalized result, you don't have to include
+          # an empty string that corresponds to the root of a given
+          # domain.
+          if TAIL_SLASH_RE =~ resource and SLASH != resource
+            normalized.push resource.sub(TAIL_SLASH_RE, "".freeze)
+          end
+
+          normalized.push resource
+        end
+      end
+    end
+  end
+end

data/lib/log_line_parser/utils.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+require 'log_line_parser'
+require 'log_line_parser/query'
+require 'csv'
+
+module LogLineParser
+  module Utils
+    TAB = "\t"
+    SPECIAL_CHARS = {
+      "\t" => '\\t',
+      "\n" => '\\n',
+      "\r" => '\\r',
+      '\\\\' => '\\\\',
+    }
+    SPECIAL_CHARS_RE = Regexp.compile(SPECIAL_CHARS.keys.join("|"))
+
+    def self.access_by_bots?(record, bots_re=Query::DEFAULT_BOTS_RE)
+      Query.access_by_bots?(record, bots_re)
+    end
+
+    def self.open_multiple_output_files(base_names, dir=nil, ext="log")
+      logs = {}
+      filepath = dir ? File.join(dir, "%s.#{ext}") : "%s.#{ext}"
+      base_names.each do |base|
+        logs[base] = open(format(filepath, base), "w")
+      end
+      yield logs
+    ensure
+      logs.each do |k, v|
+        v.close
+      end
+    end
+
+    def self.to_tsv(line, escape=true)
+      LogLineParser.parse(line).to_a.map do |field|
+        escape ? escape_special_chars(field) : field
+      end.join(TAB)
+    end
+
+    def self.to_csv(line)
+      LogLineParser.parse(line).to_a.to_csv
+    end
+
+    private
+
+    def self.escape_special_chars(field)
+      field.gsub(SPECIAL_CHARS_RE) do |char|
+        SPECIAL_CHARS[char]
+      end
+    end
+  end
+end

data/lib/log_line_parser/version.rb

@@ -0,0 +1,3 @@
+module LogLineParser
+  VERSION = "0.1.0"
+end