log_line_parser 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ce0b6e321f311f85bc375942205567ab4d60743
-  data.tar.gz: ee2d0b41508c833d39cade577581426dbb06cb23
+  metadata.gz: 658d4493353541baf8aec1350bcd803b4b11b9e9
+  data.tar.gz: 1158015fed2a075f93da9c94bcd8921734b83226
 SHA512:
-  metadata.gz: c2b363c6be57677fda905270e36f0ee23c39f9cb0915199eaad63a5fabc566a814437e92ad1706040e874e171107c3c901e90d195c1d530ea4e0282edac0d2b6
-  data.tar.gz: ea2ae4f5b9c0eb962c221e605927f0cdf977c194d612fff620a0cc2bb113825620762b67c4e377abab4654dcc1877dc3134f73625d07119d7cfa70610eb9d2a7
+  metadata.gz: 3bda73a0a7fa68a464b37e0b2287ce82206c5ba7fd973888b90935f7b6835c93f9c7aff93b7282f42f4311dbb801145353d306a5ae5ca5e047de6a01a7f19fd0
+  data.tar.gz: a89dbdf52206cfa8ca0e5752dfbac7041b689def7b9d448a98701cdd577cbd21bd7023140f31b66f85fdb33a23c8269aa3838afbcf3ce914453d605a6759c78a

data/.gitignore

@@ -12,3 +12,4 @@
 \#*\#
 /*.gem
 /*.zip
+/test/data/tmp/

data/.travis.yml

@@ -1,3 +1,12 @@
 language: ruby
 rvm:
-  - 2.1.6
+  - 2.2.0
+  - 2.3.3
+  - 2.4.0
+  - jruby-9.1.8.0
+sudo: required
+dist: trusty
+addons:
+  apt:
+    packages:
+      - haveged

data/README.md

@@ -1,7 +1,7 @@
 # LogLineParser
 
-LogLineParser is a simple parser of Apache access logs. It parses a line of Apache access log and turns it into an array of strings or a Hash object.
-And from the command line, you can use it as a conversion tool of file formats or as a filtering tool of access records.
+LogLineParser is a simple parser of Apache access log. It parses a line of Apache access log and turns it into an array of strings or a Hash object.
+And from the command line, you can use it as a conversion tool of file format (to CSV or TSV) or as a filtering tool of access records.
 
 ## Installation
 
@@ -19,6 +19,21 @@ Or install it yourself as:
 
     $ gem install log_line_parser
 
+### If you have millions of records to be parsed
+
+I recommend you to use [JRuby](http://jruby.org/) (version >= 9.1.8), because it is significantly faster than MRI.
+
+When you use JRuby, you have to set RUBYOPT environment variable at first. For example:
+
+    # For Bash or Zsh
+    $ RUBYOPT='-Xcompile.invokedynamic=true'
+    $ export RUBYOPT
+
+Or
+
+    # For MS Windows
+    set RUBYOPT=-Xcompile.invokedynamic=true
+
 ## Usage
 
 ### As a converter
@@ -63,7 +78,7 @@ Three parsers are predefined for such cases:
 <dt>LogLineParser::CommonLogWithVHParser</dt>
 <dd>For Common Log Format with Virtual Host</dd>
 <dt>LogLineParser::CombinedLogParser</dt>
-<dd>NCSA extended/combined log format</dd>
+<dd>For NCSA extended/combined log format</dd>
 </dl>
 
 #### Defining a parser
@@ -249,6 +264,21 @@ The values given by the configuration file are compiled into a regular expressio
 
     $ log_line_parser --show-current-settings --bots-config=bots_config.yml
 
+#### Command line options
+
+The following command options are available.
+
+|Short                 |Long                              |Description                                                                                                           |
+|----------------------|----------------------------------|----------------------------------------------------------------------------------------------------------------------|
+|-c [config_file]      |--config [=config_file]           |Give a configuration file in yaml format                                                                              |
+|-b [bots_config_file] |--bots-config [=bots_config_file] |Give a configuration file in yaml format. Default bots: Googlebot, Googlebot-Mobile, Mediapartners-Google, Bingbot, Slurp, Baiduspider, BaiduImagespider, BaiduMobaider, YetiBot, Applebot |
+|-s                    |--show-current-settings           |Show the detail of the current settings                                                                               |
+|-f                    |--filter-mode                     |Mode for choosing log records that satisfy certain criteria                                                           |
+|-l [LogFormat]        |--log-format [=LogFormat]         |Specify LogFormat by giving a LogFormat or one of formats predefined as "common", "common_with_vh", "combined", "moe" |
+|-o [output_dir]       |--output-dir [=output_dir]        |Specify the output directory for log files                                                                            |
+|-t [format]           |--to [=format]                    |Specify a format: csv, tsv or ltsv                                                                                    |
+|-e [error_log_file]   |--error-log [=error_log_file]     |Specify a file for error logging                                                                                      |
+
 
 ## Development
 

data/lib/log_line_parser.rb

@@ -268,8 +268,18 @@ module LogLineParser
       begin
         yield line, parser.parse(line)
       rescue MalFormedRecordError => e
-        error_output.print e.message
+        error_output.print error_message(input, e)
       end
     end
   end
+
+  def self.error_message(input, e)
+    if input == ARGF
+      "#{ARGF.filename}:#{ARGF.file.lineno}:#{e.message}"
+    else
+      e.message
+    end
+  end
+
+  private_class_method :error_message
 end

data/lib/log_line_parser/command_line_interface.rb

@@ -10,10 +10,93 @@ module LogLineParser
   module CommandLineInterface
     class UnsupportedFormatError < StandardError; end
 
+    class Converter
+      def execute(options, output=STDOUT, input=ARGF)
+        output_format = options[:format] || DEFAULT_FORMAT
+        case output_format
+        when DEFAULT_FORMAT
+          to_csv(input, output)
+        when "tsv"
+          to_tsv(input, output)
+        when "ltsv"
+          to_ltsv(input, output, options[:log_format])
+        else
+          raise UnsupportedFormatError.new(output_format)
+        end
+      end
+
+      def to_csv(input, output)
+        input.each_line do |line|
+          output.print Utils.to_csv(line.chomp)
+        end
+      end
+
+      def to_tsv(input, output)
+        input.each_line do |line|
+          output.puts Utils.to_tsv(line.chomp)
+        end
+      end
+
+      def to_ltsv(input, output, parser)
+        input.each_line do |line|
+          output.puts parser.to_ltsv(line.chomp)
+        end
+      end
+    end
+
+    class Filter
+      OptionValues = Struct.new(:configs, :bots_re, :output_log_names,
+                                :output_dir, :log_format, :error_log)
+
+      def execute(options)
+        opt = option_values(options)
+        Utils.open_multiple_output_files(opt.output_log_names,
+                                         opt.output_dir) do |logs|
+          queries = setup_queries_from_configs(opt.configs, logs, opt.bots_re)
+          LogLineParser.each_record(error_output: opt.error_log || STDERR,
+                                    parser: opt.log_format) do |line, record|
+            queries.each {|query| query.call(line, record) }
+          end
+        end
+      ensure
+        opt.error_log.close if opt.error_log
+      end
+
+      private
+
+      def option_values(options)
+        configs = Utils.load_config_file(options[:config_file])
+        bots_re = Utils.compile_bots_re_from_config_file(options[:bots_config_file])
+        error_log = open_error_log(options[:error_log_file])
+        OptionValues.new(configs,
+                         bots_re,
+                         collect_output_log_names(configs),
+                         options[:output_dir],
+                         options[:log_format],
+                         error_log)
+      end
+
+      def collect_output_log_names(configs)
+        configs.map do |config|
+          config[Query::ConfigFields::OUTPUT_LOG_NAME]
+        end
+      end
+
+      def setup_queries_from_configs(configs, logs, bots_re)
+        configs.map do |config|
+          Query.register_query_to_log(config, logs, bots_re)
+        end
+      end
+
+      def open_error_log(log_file)
+        open(File.expand_path(log_file), "wb") if log_file
+      end
+    end
+
     DEFAULT_FORMAT = "csv"
 
     def self.parse_options
-      options = {}
+      options = { log_format: LogLineParser::CombinedLogParser }
 
       OptionParser.new("USAGE: #{File.basename($0)} [OPTION]... [LOG_FILE]...") do |opt|
         opt.on("-c [config_file]", "--config [=config_file]",
@@ -40,7 +123,7 @@ Default bots: #{Bots::DEFAULT_BOTS.join(', ')}") do |config_file|
         opt.on("-l [LogFormat]", "--log-format [=LogFormat]",
                "Specify LogFormat by giving a LogFormat or one of \
 formats predefined as #{predefined_options_for_log_format}") do |log_format|
-          options[:log_format] = log_format
+          options[:log_format] = choose_log_parser(log_format)
         end
 
         opt.on("-o [output_dir]", "--output-dir [=output_dir]",
@@ -53,6 +136,11 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
           options[:format] = format
         end
 
+        opt.on("-e [error_log_file]", "--error-log [=error_log_file]",
+               "Specify a file for error logging") do |error_log_file|
+          options[:error_log_file] = error_log_file
+        end
+
         opt.parse!
       end
 
@@ -60,7 +148,6 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
     end
 
     def self.choose_log_parser(log_format)
-      return LogLineParser::CombinedLogParser unless log_format
       parser = LogLineParser::PREDEFINED_FORMATS[log_format]
       parser || LogLineParser.parser(log_format)
     end
@@ -77,33 +164,18 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
     end
 
     def self.show_settings(options)
-      bots_re = compile_bots_re_from_config_file(options[:bots_config_file])
-      parser = choose_log_parser(options[:log_format])
+      bots_re = Utils.compile_bots_re_from_config_file(options[:bots_config_file])
+      parser = options[:log_format]
       puts "The regular expression for bots: #{bots_re}"
       puts "LogFormat: #{parser.format_strings}"
     end
 
     def self.execute_as_filter(options)
-      configs = Utils.load_config_file(options[:config_file])
-      parser = choose_log_parser(options[:log_format])
-      output_dir = options[:output_dir]
-      bots_re = compile_bots_re_from_config_file(options[:bots_config_file])
-      execute_queries(configs, parser, output_dir, bots_re)
+      Filter.new.execute(options)
     end
 
     def self.execute_as_converter(options, output=STDOUT, input=ARGF)
-      output_format = options[:format] || DEFAULT_FORMAT
-      case output_format
-      when DEFAULT_FORMAT
-        convert_to_csv(input, output)
-      when "tsv"
-        convert_to_tsv(input, output)
-      when "ltsv"
-        convert_to_ltsv(input, output,
-                        choose_log_parser(options[:log_format]))
-      else
-        raise UnsupportedFormatError.new(output_format)
-      end
+      Converter.new.execute(options, output, input)
     end
 
     # private class methods
@@ -114,59 +186,6 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
         join(", ")
     end
 
-    def self.compile_bots_re_from_config_file(bots_config_file)
-      return Bots::DEFAULT_RE unless bots_config_file
-      configs = Utils.load_config_file(bots_config_file)[0]
-      Bots.compile_bots_re(configs)
-    end
-
-    def self.collect_output_log_names(configs)
-      configs.map do |config|
-        config[Query::ConfigFields::OUTPUT_LOG_NAME]
-      end
-    end
-
-    def self.execute_queries(configs, parser, output_dir, bots_re)
-      output_log_names = collect_output_log_names(configs)
-      Utils.open_multiple_output_files(output_log_names, output_dir) do |logs|
-        queries = setup_queries_from_configs(configs, logs, bots_re)
-        LogLineParser.each_record(parser: parser) do |line, record|
-          queries.each {|query| query.call(line, record) }
-        end
-      end
-    end
-
-    def self.setup_queries_from_configs(configs, logs, bots_re)
-      configs.map do |config|
-        Query.register_query_to_log(config, logs, bots_re)
-      end
-    end
-
-    def self.convert_to_csv(input, output)
-      input.each_line do |line|
-        output.print Utils.to_csv(line.chomp)
-      end
-    end
-
-    def self.convert_to_tsv(input, output)
-      input.each_line do |line|
-        output.puts Utils.to_tsv(line.chomp)
-      end
-    end
-
-    def self.convert_to_ltsv(input, output, parser)
-      input.each_line do |line|
-        output.puts parser.to_ltsv(line.chomp)
-      end
-    end
-
-    private_class_method(:predefined_options_for_log_format,
-                         :compile_bots_re_from_config_file,
-                         :collect_output_log_names,
-                         :execute_queries,
-                         :setup_queries_from_configs,
-                         :convert_to_csv,
-                         :convert_to_tsv,
-                         :convert_to_ltsv)
+    private_class_method :predefined_options_for_log_format
   end
 end

data/lib/log_line_parser/line_parser.rb

@@ -3,13 +3,17 @@
 module LineParser
   class Tokenizer
     class << self
-      attr_reader :special_token_re, :non_special_token_re
+      attr_reader :special_token_re
 
       def tokenize(str, tokens=[])
         @scanner.string = str
-        token = true # to start looping, you should assign a truthy value
-        while token
-          tokens.push token if token = scan_token
+        cur_pos = 0 # instead of @scanner.pos
+        while chunk_size = @scanner.skip_until(@special_token_re)
+          token = @scanner.matched
+          pre_match_size = chunk_size - token.bytesize
+          tokens.push str.byteslice(cur_pos, pre_match_size) if pre_match_size > 0
+          tokens.push token
+          cur_pos += chunk_size
         end
 
         tokens.push @scanner.rest unless @scanner.eos?
@@ -20,16 +24,11 @@ module LineParser
         @special_tokens = special_tokens
         @unescaped_special_tokens = unescaped_special_tokens
         @scanner = StringScanner.new("".freeze)
-        @special_token_re, @non_special_token_re = compose_re(@special_tokens)
+        @special_token_re = compose_re(@special_tokens)
       end
 
       private
 
-      def scan_token
-        @scanner.scan(@special_token_re) ||
-          @scanner.scan_until(@non_special_token_re)
-      end
-
       def compose_special_tokens_str(special_tokens)
         sorted = special_tokens.sort {|x, y| y.length <=> x.length }
         escaped = sorted.map {|token| Regexp.escape(token) }
@@ -39,7 +38,7 @@ module LineParser
 
       def compose_re(special_tokens)
         tokens_str = compose_special_tokens_str(special_tokens)
-        return Regexp.compile(tokens_str), Regexp.compile("(?=#{tokens_str})")
+        return Regexp.compile(tokens_str)
       end
     end
   end

data/lib/log_line_parser/ltsv.rb

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
 module LogLineParser
-
   module Ltsv
     LABEL_SEPARATOR = ":"
     TAB = "\t"

data/lib/log_line_parser/utils.rb

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'log_line_parser'
-require 'log_line_parser/query'
 require 'csv'
 require 'yaml'
 
@@ -24,9 +23,7 @@ module LogLineParser
       end
       yield logs
     ensure
-      logs.each do |k, v|
-        v.close
-      end
+      logs.each_value {|v| v.close }
     end
 
     def self.read_configs(config)
@@ -39,6 +36,12 @@ module LogLineParser
       end
     end
 
+    def self.compile_bots_re_from_config_file(bots_config_file)
+      return Bots::DEFAULT_RE unless bots_config_file
+      configs = load_config_file(bots_config_file)[0]
+      Bots.compile_bots_re(configs)
+    end
+
     def self.to_tsv(line, escape=true)
       LogLineParser.parse(line).to_a.map do |field|
         escape ? escape_special_chars(field) : field

data/lib/log_line_parser/version.rb

@@ -1,3 +1,3 @@
 module LogLineParser
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/log_line_parser.gemspec

@@ -10,8 +10,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["HASHIMOTO, Naoki"]
   spec.email         = ["hashimoto.naoki@gmail.com"]
 
-  spec.summary       = %q{A simple parser of Apache access logs}
-  spec.description   = %q{A simple parser of Apache access logs: it parses a line of Apache access log and turns it into an array of strings or a Hash object. And from the command line, you can use it as a conversion tool of file formats or as a filtering tool of access records.}
+  spec.summary       = %q{A simple parser of Apache access log}
+  spec.description   = %q{A simple parser of Apache access log: it parses a line of Apache access log and turns it into an array of strings or a Hash object. And from the command line, you can use it as a conversion tool of file format (to CSV/TSV) or as a filtering tool of access records.}
   spec.homepage      = "https://github.com/nico-hn/LogLineParser"
   spec.license       = "MIT"
 
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "bundler", ">= 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: log_line_parser
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - HASHIMOTO, Naoki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-10 00:00:00.000000000 Z
+date: 2017-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -38,10 +38,24 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-description: 'A simple parser of Apache access logs: it parses a line of Apache access
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'A simple parser of Apache access log: it parses a line of Apache access
   log and turns it into an array of strings or a Hash object. And from the command
-  line, you can use it as a conversion tool of file formats or as a filtering tool
-  of access records.'
+  line, you can use it as a conversion tool of file format (to CSV/TSV) or as a filtering
+  tool of access records.'
 email:
 - hashimoto.naoki@gmail.com
 executables:
@@ -99,5 +113,5 @@ rubyforge_project:
 rubygems_version: 2.2.3
 signing_key: 
 specification_version: 4
-summary: A simple parser of Apache access logs
+summary: A simple parser of Apache access log
 test_files: []