log_line_parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 86f95f21fb4df9bd1a358a43e988c4c78bdfada6
+  data.tar.gz: a14d81562c43a2f80525a436138d263816d60617
+SHA512:
+  metadata.gz: a59e7d1a346527f9a7cb32760c80d65b7a58d6b24b5b7ca28a9205028856ece88f717983261862c65c54b7afca16e02832357d846a1f9e272c576972ad7799b5
+  data.tar.gz: b6153aaec48fb5340a7445b488c73c24cb776ffb7c0f1ab937f8d48fb47b9aa87fce661b31b0c17eab4590f33bb4b48ed1e34d6d1a37499d85bf228218e47eab

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.ruby-version
+*~
+\#*\#
+/*.gem
+/*.zip

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.1.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in log_line_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 HASHIMOTO, Naoki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,216 @@
+# LogLineParser
+
+LogLineParser is a simple parser of Apache access logs. It parses a line of Apache access log and turns it into an array of strings or a Hash object.
+And from the command line, you can use it as a conversion tool of file formats or as a filtering tool of access records.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'log_line_parser'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install log_line_parser
+
+## Usage
+
+### As a converter
+
+
+```ruby
+require 'log_line_parser'
+
+line = '192.168.3.4 - - [07/Feb/2016: ... ] ...'
+LogLineParser.parse(line).to_a
+# => ["192.168.3.4", "-", "-", "07/Feb/2016: ... ", ... ]
+```
+
+Or in limited cases, parsers corresponding to  certain LogFormats are available:
+
+```ruby
+require 'log_line_parser'
+
+line = '192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"'
+LogLineParser::CombinedLogParser.to_hash(line)
+# => {
+#   "%h" => "192.168.3.4",
+#   "%l" => "-",
+#   "%u" => "quidam",
+#   "%t" => "07/Feb/2016:07:39:42 +0900",
+#   "%r" => "GET /index.html HTTP/1.1",
+#   "%>s" => "200",
+#   "%b" => "432",
+#   "%{Referer}i" => "http://www.example.org/start.html",
+#   "%{User-agent}i" => "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0",
+#   "%m" => "GET",
+#   "%H" => "HTTP/1.1",
+#   "%U%q" => "/index.html"
+# }
+```
+
+Three parsers are predefined for such cases:
+
+<dl>
+<dt>LogLineParser::CommonLogParser</dt>
+<dd>For Common Log Format (CLF)</dd>
+<dt>LogLineParser::CommonLogWithVHParser</dt>
+<dd>For Common Log Format with Virtual Host</dd>
+<dt>LogLineParser::CombinedLogParser</dt>
+<dd>NCSA extended/combined log format</dd>
+</dl>
+
+#### Defining a parser
+
+You can define your own parser as in the following example:
+
+```ruby
+require 'log_line_parser'
+
+RefererLogParser = LogLineParser.parser('"%r" %>s %b %{Referer}i -> %U')
+
+line = '"GET /index.html HTTP/1.1" 200 432 http://www.example.org/start.html -> /index.html'
+
+RefererLogParser.to_hash(line)
+# => {
+#   "%r" => "GET /index.html HTTP/1.1",
+#   "%>s" => "200",
+#   "%b" => "432",
+#   "%{Referer}i" => "http://www.example.org/start.html",
+#   "->" => "->",
+#   "%U" => "/index.html",
+#   "%m" => "GET",
+#   "%H" => "HTTP/1.1",
+#   "%U%q" => "/index.html"
+# }
+```
+
+#### Limitations
+
+* Currently, you should include at least `%r`, `%>s` and `%b` in the format strings passed to `LogLineParser.parser`.
+* If the value of a field is expected to contain a space, such field should be enclosed in double quotes (that means you have to change access log settings).
+
+### As a command-line application
+
+The command line tool `log_line_parser` can be used for two purposes:
+
+1. For converting file formats
+2. For picking up log records that satisfy certain criteria
+
+For the first purpose, the tool support conversion from an Apache log format to CSV or TSV format.
+And for the second purpose, criteria such as :not_found?(= :status_code_404?) or :access_by_bots? are defined, and you can combine them by writing a configuration file.
+
+#### For converting file formats
+
+Suppose you have an Apache log file [example_combined_log.log](./test/data/example_combined_log.log), and run the following command in your terminal:
+
+    $ log_line_parser example_combined_log.log > expected_combined_log.csv
+
+Then you will get [expected_combined_log.csv](./test/data/expected_combined_log.csv).
+
+To convert into TSV format:
+
+    $ log_line_parser --to=tsv example_combined_log.log > expected_combined_log.tsv
+
+And you will get [expected_combined_log.tsv](./test/data/expected_combined_log.tsv).
+
+#### For picking up log records
+
+First, you have to prepare a configuration file in YAML format. [samples/sample_config.yml](./samples/sample_config.yml) is an example.
+
+Second, run the following command if you want to pick up from [samples/sample_combined_log.log](./samples/sample_combined_log.log) the log records that meet the definitions in the configuration file:
+
+    $ log_line_parser --filter-mode --log-format combined --config=samples/sample_config.yml --output-dir=samples/output samples/sample_combined_log.log
+
+Then the results are in [samples/output](https://github.com/nico-hn/LogLineParser/tree/master/samples/output/) directory.
+
+##### Format of configuration
+
+An example of configurations is below:
+
+```yaml
+---
+host_name: www.example.org
+resources:
+ - /end.html
+ - /subdir/big.pdf
+match:
+ - :access_to_resources?
+match_type: any
+output_log_name: access-to-two-specific-files
+---
+host_name: www.example.org
+resources:
+ - /
+match:
+ - :access_to_under_resources?
+match_type: any
+ignore_match:
+ - :access_by_bots?
+ - :not_found?
+output_log_name: all-but-bots-and-not-found
+---
+host_name: www.example.org
+resources:
+ - /index.html
+match:
+ - :access_to_resources?
+ - :access_by_bots?
+match_type: all
+output_log_name: index-page-accessed-by-bot
+```
+It contains three configurations, and each of them consists of parameters in the following table:
+
+|Parameters              |Note                                                                                                       |
+|------------------------|-----------------------------------------------------------------------------------------------------------|
+|host_name (optional)    |Currently, the specified value is compared with the host part of the value of "%{Referer}i".               |
+|resources               |The values will be compared with the value of "%U%q" field or the path part of the value of "%{Referer}i". |
+|match                   |The criteria that a log record should satisfy.                                                             |
+|ignore_match (optional) |If a log record satisfies any of the criteria listed under this parameter, the record is ignored.          |
+|match_type (optional)   |The value is "any" (default) or "all". "any" means a log record is picked up if any of the criteria listed under the "match" parameter is satisfied. "all" means all of the criteria must be satisfied for the picking up. |
+|output_log_name         |Log records picked up are written in the file specified by this parameter.                                 |
+
+
+##### Criteria for "match" and "ignore_match" parameters
+
+|Available criteria                      |Note                                                                                      |
+|----------------------------------------|------------------------------------------------------------------------------------------|
+|:access_by_bots?                        |Access by major web crawlers such as Googlebot or Bingbot.                                |
+|:referred_from_resources?               |The path part of the value of "%{Referer}i" matches any of the values of "resources".     |
+|:referred_from_under_resources?         |The path part of the value of "%{Referer}i" begins with any of the values of "resources". |
+|:access_to_resources?                   |The value of "%U%q" matches any of the values of "resources".                             |
+|:access_to_under_resources?             |The value of "%U%q" begins with any of the values of "resources".                         |
+|:partial_content? / :status_code_206?   |The value of "%>s" is 206.                                                                |
+|:moved_permanently? / :status_code_301? |The value of "%>s" is 301.                                                                |
+|:not_modified? / :status_code_304?      |The value of "%>s" is 304.                                                                |
+|:not_found? / :status_code_404?         |The value of "%>s" is 404.                                                                |
+|:options_method?                        |The value of "%m" is OPTIONS                                                              |
+|:get_method?                            |The value of "%m" is GET.                                                                 |
+|:head_method?                           |The value of "%m" is HEAD.                                                                |
+|:post_method?                           |The value of "%m" is POST.                                                                |
+|:put_method?                            |The value of "%m" is PUT.                                                                 |
+|:delete_method?                         |The value of "%m" is DELETE.                                                              |
+|:trace_method?                          |The value of "%m" is TRACE.                                                               |
+|:connect_method?                        |The value of "%m" is CONNECT.                                                             |
+|:patch_method?                          |The value of "%m" is PATCH.                                                               |
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec log_line_parser` to use the code located in this directory, ignoring other installed copies of this gem.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/nico-hn/LogLineParser/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs = ["lib", "test"]
+  t.warning = true
+  t.verbose = true
+  t.test_files = FileList['test/test_*.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "log_line_parser"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/log_line_parser

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "log_line_parser"
+require "log_line_parser/command_line_interface"
+require "log_line_parser/moe"
+
+LogLineParser::CommandLineInterface.execute

data/lib/log_line_parser/apache.rb

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+module LogLineParser
+  module Apache
+=begin
+All of the format strings listed in http://httpd.apache.org/docs/current/mod/mod_log_config.html#formats:
+%% %a %{c}a %A %B %b %{VARNAME}C %D %{VARNAME}e %f %h %H %{VARNAME}i %k %l %L %m %{VARNAME}n %{VARNAME}o %p %{format}p %P %{format}P %q %r %R %s %t %{format}t %T %{UNIT}T %u %U %v %V %X %I %O %S %{VARNAME}^ti %{VARNAME}^to
+
+As explained in http://httpd.apache.org/docs/current/logs.html:
+"%r" = "%m %U%q %H"
+=end
+
+    module LogFormat
+      COMMON = "%h %l %u %t \"%r\" %>s %b"
+      COMMON_WITH_VH = "%v %h %l %u %t \"%r\" %>s %b"
+      COMBINED = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\""
+    end
+
+    FORMAT_STRING_SYMBOLE_TABLE = {
+      "%%" => :percent,
+      "%a" => :remote_ip,
+      "%{c}a" => :underlying_peer_ip,
+      "%A" => :local_ip,
+      "%B" => :response_bytes,
+      "%b" => :response_bytes,
+      # "%{VARNAME}C" => :cookie,
+      "%D" => :time_taken_us,
+      # "%{VARNAME}e" => :,
+      "%f" => :filename,
+      "%h" => :remote_host,
+      "%H" => :protocol,
+      # "%{VARNAME}i" => :,
+      "%{Referer}i" => :referer,
+      "%{User-agent}i" => :user_agent,
+      "%{X-Forwarded-For}i" => :x_forwarded_for,
+      "%k" => :keepalive_number,
+      "%l" => :remote_logname,
+      "%L" => :error_log_request_id,
+      "%m" => :method,
+      # "%{VARNAME}n" => :,
+      # "%{VARNAME}o" => :,
+      "%p" => :server_port,
+      # "%{format}p" => :,
+      "%P" => :pid,
+      # "%{format}P" => :,
+      "%q" => :query_string,
+      "%r" => :first_line_of_request,
+      # "%R" => :handler,
+      "%s" => :original_request_status,
+      "%>s" => :last_request_status, # final status
+      "%t" => :time, # Time the request was received
+      # "%{format}t" => :,
+      "%T" => :time_taken_s,
+      # "%{UNIT}T" => :,
+      "%u" => :remote_user,
+      "%U" => :url_path,
+      "%U%q" => :resource,
+      "%v" => :virtual_host,
+      "%V" => :server_name2,
+      "%X" => :connection_status,
+      "%I" => :received_bytes_including_headers,
+      "%O" => :sent_bytes_including_headers,
+      "%S" => :bytes_transferred,
+      # "%{VARNAME}^ti" => :,
+      # "%{VARNAME}^to" => :,
+    }
+
+    def self.parse_log_format(log_format)
+      log_format.split(/ /).map do |string|
+        string.sub(/^"/, "".freeze).sub(/"$/, "".freeze)
+      end
+    end
+
+    def self.format_strings_to_symbols(format_strings)
+      format_strings.map do |string|
+        FORMAT_STRING_SYMBOLE_TABLE[string]||string.to_sym
+      end
+    end
+  end
+end

data/lib/log_line_parser/command_line_interface.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'optparse'
+require 'log_line_parser'
+require 'log_line_parser/query'
+require 'log_line_parser/utils'
+
+module LogLineParser
+  module CommandLineInterface
+    class UnsupportedFormatError < StandardError; end
+
+    DEFAULT_FORMAT = "csv"
+
+    def self.read_configs(config)
+      YAML.load_stream(config).to_a
+    end
+
+    def self.parse_options
+      options = {}
+
+      OptionParser.new("USAGE: #{File.basename($0)} [OPTION]... [LOG_FILE]...") do |opt|
+        opt.on("-c [config_file]", "--config [=config_file]",
+               "Give a configuration file in yaml format") do |config_file|
+          options[:config_file] = config_file
+        end
+
+        opt.on("-f", "--filter-mode",
+               "Mode for choosing log records that satisfy certain criteria") do
+          options[:filter_mode] = true
+        end
+
+        opt.on("-l [LogFormat]", "--log-format [=LogFormat]",
+               "Specify LogFormat") do |log_format|
+          options[:log_format] = log_format
+        end
+
+        opt.on("-o [output_dir]", "--output-dir [=output_dir]",
+               "Specify the output directory for log files") do |output_dir|
+          options[:output_dir] = output_dir
+        end
+
+        opt.on("-t [format]", "--to [=format]",
+               "Specify a format") do |format|
+          options[:format] = format
+        end
+
+        opt.parse!
+      end
+
+      options
+    end
+
+    def self.load_config_file(config_file)
+      open(File.expand_path(config_file)) do |f|
+        read_configs(f.read)
+      end
+    end
+
+    def self.choose_log_parser(log_format)
+      return LogLineParser::CombinedLogParser unless log_format
+      parser = LogLineParser::PREDEFINED_FORMATS[log_format]
+      parser || LogLineParser.parser(log_format)
+    end
+
+    def self.execute
+      options = parse_options
+      if options[:filter_mode]
+        execute_as_filter(options)
+      else
+        execute_as_converter(options)
+      end
+    end
+
+    def self.execute_as_filter(options)
+      configs = load_config_file(options[:config_file])
+      parser = choose_log_parser(options[:log_format])
+      output_dir = options[:output_dir]
+      output_log_names = collect_output_log_names(configs)
+      Utils.open_multiple_output_files(output_log_names, output_dir) do |logs|
+        queries = setup_queries_from_configs(configs, logs)
+        LogLineParser.each_record(record_type: parser) do |line, record|
+          queries.each {|query| query.call(line, record) }
+        end
+      end
+    end
+
+    def self.execute_as_converter(options, output=STDOUT, input=ARGF)
+      output_format = options[:format] || DEFAULT_FORMAT
+      case output_format
+      when DEFAULT_FORMAT
+        convert_to_csv(input, output)
+      when "tsv"
+        convert_to_tsv(input, output)
+      else
+        raise UnsupportedFormatError.new(output_format)
+      end
+    end
+
+    private
+
+    def self.collect_output_log_names(configs)
+      configs.map do |config|
+        config[Query::ConfigFields::OUTPUT_LOG_NAME]
+      end
+    end
+
+    def self.setup_queries_from_configs(configs, logs)
+      configs.map do |config|
+        Query.register_query_to_log(config, logs)
+      end
+    end
+
+    def self.convert_to_csv(input, output)
+      input.each_line do |line|
+        output.print Utils.to_csv(line.chomp)
+      end
+    end
+
+    def self.convert_to_tsv(input, output)
+      input.each_line do |line|
+        output.puts Utils.to_tsv(line.chomp)
+      end
+    end
+  end
+end