log_line_parser 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86f95f21fb4df9bd1a358a43e988c4c78bdfada6
-  data.tar.gz: a14d81562c43a2f80525a436138d263816d60617
+  metadata.gz: 98ed47b6f1624564237a5fcd95df8fa59f7e68d4
+  data.tar.gz: 9236e365d98e5a70e3503956327ca28285d2ff0f
 SHA512:
-  metadata.gz: a59e7d1a346527f9a7cb32760c80d65b7a58d6b24b5b7ca28a9205028856ece88f717983261862c65c54b7afca16e02832357d846a1f9e272c576972ad7799b5
-  data.tar.gz: b6153aaec48fb5340a7445b488c73c24cb776ffb7c0f1ab937f8d48fb47b9aa87fce661b31b0c17eab4590f33bb4b48ed1e34d6d1a37499d85bf228218e47eab
+  metadata.gz: 2af2324f7fd1b56dc88c65a68b618ec7c41a8e30e2054e59c5f96a2bf2c3db1d167998f8b8baa0bcd13f783b3ee6dfdbf0b0d4ec7f0f934434a22441cb76607e
+  data.tar.gz: 8301442589177b2caa33c3e8ecea442f7e8abb215dabdbd746f9020d4cfca0334d2f153a3d0194d9664826043bcfa129bc3be48ebae2b0bfcbe456f63f62c03e

data/README.md

@@ -28,7 +28,7 @@ Or install it yourself as:
 require 'log_line_parser'
 
 line = '192.168.3.4 - - [07/Feb/2016: ... ] ...'
-LogLineParser.parse(line).to_a
+LogLineParser.to_array(line)
 # => ["192.168.3.4", "-", "-", "07/Feb/2016: ... ", ... ]
 ```
 
@@ -103,7 +103,7 @@ The command line tool `log_line_parser` can be used for two purposes:
 1. For converting file formats
 2. For picking up log records that satisfy certain criteria
 
-For the first purpose, the tool support conversion from an Apache log format to CSV or TSV format.
+For the first purpose, the tool supports conversion from an Apache log format to CSV or TSV format.
 And for the second purpose, criteria such as :not_found?(= :status_code_404?) or :access_by_bots? are defined, and you can combine them by writing a configuration file.
 
 #### For converting file formats

data/lib/log_line_parser/command_line_interface.rb

@@ -31,7 +31,8 @@ module LogLineParser
         end
 
         opt.on("-l [LogFormat]", "--log-format [=LogFormat]",
-               "Specify LogFormat") do |log_format|
+               "Specify LogFormat by giving a LogFormat or one of \
+formats predefined as #{predefined_options_for_log_format}") do |log_format|
           options[:log_format] = log_format
         end
 
@@ -41,7 +42,7 @@ module LogLineParser
         end
 
         opt.on("-t [format]", "--to [=format]",
-               "Specify a format") do |format|
+               "Specify a format: csv, tsv or ltsv") do |format|
           options[:format] = format
         end
 
@@ -79,7 +80,7 @@ module LogLineParser
       output_log_names = collect_output_log_names(configs)
       Utils.open_multiple_output_files(output_log_names, output_dir) do |logs|
         queries = setup_queries_from_configs(configs, logs)
-        LogLineParser.each_record(record_type: parser) do |line, record|
+        LogLineParser.each_record(parser: parser) do |line, record|
           queries.each {|query| query.call(line, record) }
         end
       end
@@ -92,6 +93,9 @@ module LogLineParser
         convert_to_csv(input, output)
       when "tsv"
         convert_to_tsv(input, output)
+      when "ltsv"
+        convert_to_ltsv(input, output,
+                        choose_log_parser(options[:log_format]))
       else
         raise UnsupportedFormatError.new(output_format)
       end
@@ -99,6 +103,12 @@ module LogLineParser
 
     private
 
+    def self.predefined_options_for_log_format
+      PREDEFINED_FORMATS.keys.
+        map {|opt| "\"#{opt}\"" }.
+        join(", ")
+    end
+
     def self.collect_output_log_names(configs)
       configs.map do |config|
         config[Query::ConfigFields::OUTPUT_LOG_NAME]
@@ -122,5 +132,11 @@ module LogLineParser
         output.puts Utils.to_tsv(line.chomp)
       end
     end
+
+    def self.convert_to_ltsv(input, output, parser)
+      input.each_line do |line|
+        output.puts parser.to_ltsv(line.chomp)
+      end
+    end
   end
 end

data/lib/log_line_parser/ltsv.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+module LogLineParser
+
+  module Ltsv
+    LABEL_SEPARATOR = ":"
+    TAB = "\t"
+
+    ##
+    # Label names are borrowed from
+    # http://ltsv.org/
+
+    FORMAT_STRING_LABEL_TABLE = {
+      "%t" => "time",
+      "%h" => "host",
+      "%{X-Forwarded-For}i" => "forwardedfor",
+      "%l" => "ident",
+      "%u" => "user",
+      "%r" => "req",
+      "%m" => "method",
+      "%U%q" => "uri",
+      "%H" => "protocol",
+      "%>s" => "status",
+      "%B" => "size",
+      "%b" => "size",
+      "%I" => "reqsize",
+      "%{Referer}i" => "referer",
+      "%{User-agent}i" => "ua",
+      "%{Host}i" => "vhost",
+      "%D" => "reqtime_microsec",
+      "%T" => "reqtime",
+      "%{X-Cache}o" => "cache",
+      "%{X-Runtime}o" => "runtime",
+      # "-" => "apptime",
+    }
+
+    def self.format_strings_to_labels(format_strings)
+      format_strings.map do |string|
+        FORMAT_STRING_LABEL_TABLE[string]||string
+      end
+    end
+
+    def self.to_ltsv(labels, values)
+      fields = labels.zip(values).map {|field| field.join(LABEL_SEPARATOR) }
+      fields.join(TAB)
+    end
+  end
+end

data/lib/log_line_parser/moe.rb

@@ -3,16 +3,21 @@
 require 'log_line_parser'
 require 'log_line_parser/utils'
 
-# MoeLogParser is added from the personal needs of the original author,
-# and the LogFormat for it is not a widely used format.
-# You may remove this file if you don't need it.
-# (MOE is the acronym of the organization's name for which the author
-# is working at the time of the first release of this program.)
-
 module LogLineParser
-  # CombinedLogFormat + "%D"
+
+  # MoeLogFormat and MoeLogParser is added from the personal needs of the
+  # original author, and the log format is not a widely used one.
+  # You may remove this file if you don't need it.
+  # (MOE is the acronym of the name of the organization for which
+  # the author is working at the time of the first release of this program.)
+  #
+  # MoeLogFormat = CombinedLogFormat + "%D"
   MoeLogFormat = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D"
+
+  ##
+  # Parser of MoeLogFormat
   MoeLogParser = parser(MoeLogFormat)
+
   PREDEFINED_FORMATS['moe'] = MoeLogParser
 end
 

data/lib/log_line_parser/query.rb

@@ -75,6 +75,10 @@ YetiBot
       bots_re =~ record.user_agent
     end
 
+    ##
+    # Returns true if the path+query part of the value of %{Referer}i
+    # matchs one of resources.
+
     def self.referred_from_resources?(record, resources=[])
       resources.include?(record.referer_resource)
     end
@@ -183,11 +187,50 @@ YetiBot
       bots_re =~ record.user_agent
     end
 
+    ##
+    # Returns true if the path+query part of the value of %{Referer}i
+    # matches one of the resources that are passed as the second
+    # argument when you create an instance of Query.
+    #
+    # When a given resource is a directory, you should append a "/" at the
+    # end of it, otherwise you would get a wrong result. For example,
+    # suppose you define the following queries:
+    #
+    #    correct_query = Query.new("www.example.org", ["/dir/subdir/"])
+    #    wrong_query = Query.new("www.example.org", ["/dir/subdir"])
+    #
+    # <tt>correct_query.referred_from_resources?(record)</tt> returns true
+    # when the value of %{Referer}i is "http://www.example.org/subdir"
+    # or "http://www.example.org/subdir/",
+    # but <tt>wrong_query.referred_from_resources?(record)</tt> returns
+    # false for "http://www.example.org/subdir/"
+
     def referred_from_resources?(record)
       if_matching_domain(record) and
         @normalized_resources.include?(record.referer_resource)
     end
 
+    ##
+    # Returns true if the path+query part of the value of %{Referer}i
+    # begins with one of the resources that are passed as the second
+    # argument when you create an instance of Query.
+    #
+    # When a given resource is a directory, you should append a "/" at the
+    # end of it, otherwise you would get a wrong result. For example,
+    # suppose you define the following queries:
+    #
+    #    correct_query = Query.new("www.example.org", ["/dir/subdir/"])
+    #    wrong_query = Query.new("www.example.org", ["/dir/subdir"])
+    #
+    # <tt>wrong_query.referred_from_under_resources?(record)</tt>
+    # returns true even when the value of %{Referer}i in record is
+    # "http://www.example.org/subdir_for_images/a_file_name",
+    # while <tt>correct_query.referred_from_under_resources?(record)</tt>
+    # returns true when the value of %{Referer}i is
+    # "http://www.example.org/subdir/a_filename" or
+    # "http://www.example.org/subdir",
+    # and returns false for "http://www.example.org/subdir_for_images".
+
     def referred_from_under_resources?(record)
       referer_resource = record.referer_resource
       if_matching_domain(record) and
@@ -195,10 +238,51 @@ YetiBot
         @resources.any?{|target| referer_resource.start_with?(target) }
     end
 
+    ##
+    # Returns true if the value of %U%q in +record+ matches one of the
+    # resources that are passed as the second argument when you create
+    # an instance of Query.
+    #
+    # When you give a directory as one of resources, you should append
+    # a "/" at the end of the directory, otherwise records whose %U%q
+    # value points to the same directory but without trailing "/"
+    # will return false.
+    #
+    # For example, when you create queries as follows,
+    #
+    #    query_with_slash = Query.new("www.example.org", ["/dir/subdir/"])
+    #    query_without_slash = Query.new("www.example.org", ["/dir/subdir"])
+    #
+    # <tt>query_with_slash.access_to_resources?(record)</tt> returns true for
+    # both of records whose %U%q value is "/dir/subdir/" and "/dir/subdir"
+    # respectively.
+    #
+    # But <tt>query_without_slash.access_to_resources?(record)</tt> returns
+    # false for a record whose %U%q value is "/dir/subdir/"
+
     def access_to_resources?(record)
       @normalized_resources.include?(record.resource)
     end
 
+    ##
+    # Returns true if the value of %U%q in +record+ begins with one
+    # of the resources that are passed as the second argument when
+    # you create an instance of Query.
+    #
+    # When a given resource is a directory, you should append a "/" at the
+    # end of it, otherwise you would get a wrong result. For example,
+    # suppose you define the following queries:
+    #
+    #    correct_query = Query.new("www.example.org", ["/dir/subdir/"])
+    #    wrong_query = Query.new("www.example.org", ["/dir/subdir"])
+    #
+    # <tt>wrong_query.access_to_under_resources?(record)</tt>
+    # returns true even when the value of %U%q in record is
+    # "/subdir_for_images/a_file_name", while
+    # <tt>correct_query.access_to_under_resources?(record)</tt>
+    # returns true when the value of %U%q is "/subdir/a_filename" or
+    # "/subdir", and returns false for "/subdir_for_images".
+
     def access_to_under_resources?(record)
       resource = record.resource
       @normalized_dirs.include?(resource) or

data/lib/log_line_parser/version.rb

@@ -1,3 +1,3 @@
 module LogLineParser
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/log_line_parser.rb

@@ -3,6 +3,7 @@
 require "log_line_parser/version"
 require "log_line_parser/line_parser"
 require "log_line_parser/apache"
+require "log_line_parser/ltsv"
 require "strscan"
 require "time"
 require "date"
@@ -65,12 +66,12 @@ module LogLineParser
       root.subnodes.map {|node| node.to_s }
     end
 
-    def to_hash(record_type=CombinedLogParser)
-      record_type.to_hash(to_a)
+    def to_hash(parser=CombinedLogParser)
+      parser.to_hash(to_a)
     end
 
-    def to_record(record_type=CombinedLogParser)
-      record_type.create(to_a)
+    def to_record(parser=CombinedLogParser)
+      parser.create(to_a)
     end
   end
 
@@ -82,6 +83,7 @@ module LogLineParser
     def setup(field_names, format_strings=nil)
       @field_names = field_names
       @format_strings = format_strings
+      @ltsv_labels = Ltsv.format_strings_to_labels(format_strings)
       @number_of_fields = field_names.length
       @referer_defined = field_names.include?(:referer)
       @parse_time_value = false
@@ -115,6 +117,13 @@ module LogLineParser
       h
     end
 
+    def to_ltsv(line)
+      values = line.kind_of?(Array) ? line : LogLineParser.parse(line).to_a
+      Ltsv.to_ltsv(@ltsv_labels, values)
+    end
+
+    private
+
     def parse_request(h)
       if first_line_of_request = h["%r".freeze]
         request = first_line_of_request.split(/ /)
@@ -124,8 +133,6 @@ module LogLineParser
       end
     end
 
-    private
-
     def response_size(rec)
       size_str = rec.response_bytes
       size_str == "-".freeze ? 0 : size_str.to_i
@@ -183,6 +190,17 @@ module LogLineParser
     record_type
   end
 
+  private_class_method :create_record_type
+
+  ##
+  # Creates a parser from a LogFormat.
+  #
+  # For example,
+  #
+  #    parser = LogLineParse.parser("%h %l %u %t \"%r\" %>s %b")
+  #
+  # creates the parser of Common Log Format.
+
   def self.parser(log_format)
     if log_format.kind_of? String
       format_strings = Apache.parse_log_format(log_format)
@@ -206,24 +224,49 @@ module LogLineParser
     # LogLineTokenizer.tokenize(line.chomp, stack)
   end
 
+  ##
+  # Turns a line of Apache access logs into an array of field values.
+  #
+  # Escaped characters such as "\\t" or "\\"" will be unescaped.
+
   def self.to_array(line)
     parse(line).to_a
   end
 
+  ##
+  # Parser of Common Log Format (CLF)
+  #
+  # ref: https://www.w3.org/Daemon/User/Config/Logging.html#common-logfile-format
+
   CommonLogParser = parser(Apache::LogFormat::COMMON)
+
+  ##
+  # Parser of Common Log Format with Virtual Host
+
   CommonLogWithVHParser = parser(Apache::LogFormat::COMMON_WITH_VH)
+
+  ##
+  # Parser of NCSA extended/combined log format
+
   CombinedLogParser = parser(Apache::LogFormat::COMBINED)
 
   PREDEFINED_FORMATS['common'] = CommonLogParser
   PREDEFINED_FORMATS['common_with_vh'] = CommonLogWithVHParser
   PREDEFINED_FORMATS['combined'] = CombinedLogParser
 
-  def self.each_record(record_type: CommonLogParser,
+  ##
+  # Reads each line from +input+ (Apache access log files are expected) and
+  # parses it, then yields the line and the parsed result (+record+) to the
+  # associated block.
+  #
+  # When it fails to parse a line, the line will be printed to +error_output+
+
+  def self.each_record(parser: CombinedLogParser,
                        input: ARGF,
-                       error_output: STDERR)
+                       error_output: STDERR) # :yields: line, record
     input.each_line do |line|
       begin
-        yield line, record_type.parse(line)
+        yield line, parser.parse(line)
       rescue MalFormedRecordError => e
         error_output.print e.message
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_line_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - HASHIMOTO, Naoki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-06 00:00:00.000000000 Z
+date: 2016-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -62,6 +62,7 @@ files:
 - lib/log_line_parser/apache.rb
 - lib/log_line_parser/command_line_interface.rb
 - lib/log_line_parser/line_parser.rb
+- lib/log_line_parser/ltsv.rb
 - lib/log_line_parser/moe.rb
 - lib/log_line_parser/query.rb
 - lib/log_line_parser/utils.rb