log_line_parser 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98ed47b6f1624564237a5fcd95df8fa59f7e68d4
-  data.tar.gz: 9236e365d98e5a70e3503956327ca28285d2ff0f
+  metadata.gz: 6ce0b6e321f311f85bc375942205567ab4d60743
+  data.tar.gz: ee2d0b41508c833d39cade577581426dbb06cb23
 SHA512:
-  metadata.gz: 2af2324f7fd1b56dc88c65a68b618ec7c41a8e30e2054e59c5f96a2bf2c3db1d167998f8b8baa0bcd13f783b3ee6dfdbf0b0d4ec7f0f934434a22441cb76607e
-  data.tar.gz: 8301442589177b2caa33c3e8ecea442f7e8abb215dabdbd746f9020d4cfca0334d2f153a3d0194d9664826043bcfa129bc3be48ebae2b0bfcbe456f63f62c03e
+  metadata.gz: c2b363c6be57677fda905270e36f0ee23c39f9cb0915199eaad63a5fabc566a814437e92ad1706040e874e171107c3c901e90d195c1d530ea4e0282edac0d2b6
+  data.tar.gz: ea2ae4f5b9c0eb962c221e605927f0cdf977c194d612fff620a0cc2bb113825620762b67c4e377abab4654dcc1877dc3134f73625d07119d7cfa70610eb9d2a7

data/README.md

@@ -130,7 +130,7 @@ Second, run the following command if you want to pick up from [samples/sample_co
 
 Then the results are in [samples/output](https://github.com/nico-hn/LogLineParser/tree/master/samples/output/) directory.
 
-##### Format of configuration
+##### Format of configuration file
 
 An example of configurations is below:
 
@@ -182,6 +182,7 @@ It contains three configurations, and each of them consists of parameters in the
 |Available criteria                      |Note                                                                                      |
 |----------------------------------------|------------------------------------------------------------------------------------------|
 |:access_by_bots?                        |Access by major web crawlers such as Googlebot or Bingbot.                                |
+|:access_to_image?                       |The value of "%U%q" matches /\.(?:jpe?g\|png\|gif\|ico\|tiff?\|bmp\|svgz?\|webp)$/in             |
 |:referred_from_resources?               |The path part of the value of "%{Referer}i" matches any of the values of "resources".     |
 |:referred_from_under_resources?         |The path part of the value of "%{Referer}i" begins with any of the values of "resources". |
 |:access_to_resources?                   |The value of "%U%q" matches any of the values of "resources".                             |
@@ -201,6 +202,54 @@ It contains three configurations, and each of them consists of parameters in the
 |:patch_method?                          |The value of "%m" is PATCH.                                                               |
 
 
+##### Default web crawlers
+
+The following web crawlers are set by default and used by Query#access_by_bots?:
+
+* Googlebot
+* Googlebot-Mobile
+* Mediapartners-Google
+* Bingbot
+* Slurp
+* Baiduspider
+* BaiduImagespider
+* BaiduMobaider
+* YetiBot
+* Applebot
+
+
+##### Format of bots configuration file
+
+You can specify web crawlers by giving a configuration file to `--bots-config` option. The following is an example of configuration file:
+
+```yaml
+inherit_default_bots: false
+bots:
+ - Googlebot
+ - Googlebot-Mobile
+ - Mediapartners-Google
+ - Bingbot
+ - Slurp
+ - Baiduspider
+ - BaiduImagespider
+ - BaiduMobaider
+ - YetiBot
+ - Applebot
+bots_re:
+ - " bot$"
+```
+
+|Parameters                      |Note                                                                                                                         |
+|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+|bots (optional)                 |Names of web crawlers that make Query#access_by_bots? return true when they are included in the value of "%{Referer}i".      |
+|inherit_default_bots (optional) |If this option is set to true, the default names of major web crawlers are added to the names specified by `bots` parameter. |
+|bots_re (optional)              |Use this parameter if you want to identify bots by regular expressions.                                                      |
+
+The values given by the configuration file are compiled into a regular expression, and you can check the expression by invoking the tool as follows:
+
+    $ log_line_parser --show-current-settings --bots-config=bots_config.yml
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec log_line_parser` to use the code located in this directory, ignoring other installed copies of this gem.

data/lib/log_line_parser/bots.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+module LogLineParser
+  module Bots
+    module ConfigLabels
+      INHERIT_DEFAULT_BOTS = "inherit_default_bots"
+      BOTS = "bots"
+      BOTS_RE = "bots_re"
+    end
+
+    DEFAULT_BOTS = %w(
+Googlebot
+Googlebot-Mobile
+Mediapartners-Google
+Bingbot
+Slurp
+Baiduspider
+BaiduImagespider
+BaiduMobaider
+YetiBot
+Applebot
+)
+
+    DEFAULT_CONFIG = {
+      ConfigLabels::INHERIT_DEFAULT_BOTS => true,
+      ConfigLabels::BOTS => [],
+      ConfigLabels::BOTS_RE => nil
+    }
+
+    def self.compile_bots_re(bots_config=DEFAULT_CONFIG)
+      escaped_re = compile_escaped_re(bots_config)
+      re = compile_re(bots_config)
+      return Regexp.union(escaped_re, re) if escaped_re && re
+      escaped_re || re
+    end
+
+    def self.compile_escaped_re(bots_config)
+      bot_names = bots_config[ConfigLabels::BOTS] || []
+      if bots_config[ConfigLabels::INHERIT_DEFAULT_BOTS]
+        bot_names = (DEFAULT_BOTS + bot_names).uniq
+      end
+      return if bot_names.empty?
+      escaped_bots_str = bot_names.map {|name| Regexp.escape(name) }.join("|")
+      Regexp.compile(escaped_bots_str, Regexp::IGNORECASE, "n")
+    end
+
+    def self.compile_re(bots_config)
+      bots_pats = bots_config[ConfigLabels::BOTS_RE]
+      Regexp.compile(bots_pats.join("|"), nil, "n") if bots_pats
+    end
+
+    private_class_method :compile_escaped_re, :compile_re
+
+    DEFAULT_RE = compile_bots_re
+  end
+end

data/lib/log_line_parser/command_line_interface.rb

@@ -12,10 +12,6 @@ module LogLineParser
 
     DEFAULT_FORMAT = "csv"
 
-    def self.read_configs(config)
-      YAML.load_stream(config).to_a
-    end
-
     def self.parse_options
       options = {}
 
@@ -25,6 +21,17 @@ module LogLineParser
           options[:config_file] = config_file
         end
 
+        opt.on("-b [bots_config_file]", "--bots-config [=bots_config_file]",
+               "Give a configuration file in yaml format. \
+Default bots: #{Bots::DEFAULT_BOTS.join(', ')}") do |config_file|
+          options[:bots_config_file] = config_file
+        end
+
+        opt.on("-s", "--show-current-settings",
+               "Show the detail of the current settings") do
+          options[:show_settings] = true
+        end
+
         opt.on("-f", "--filter-mode",
                "Mode for choosing log records that satisfy certain criteria") do
           options[:filter_mode] = true
@@ -52,12 +59,6 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
       options
     end
 
-    def self.load_config_file(config_file)
-      open(File.expand_path(config_file)) do |f|
-        read_configs(f.read)
-      end
-    end
-
     def self.choose_log_parser(log_format)
       return LogLineParser::CombinedLogParser unless log_format
       parser = LogLineParser::PREDEFINED_FORMATS[log_format]
@@ -66,24 +67,28 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
 
     def self.execute
       options = parse_options
-      if options[:filter_mode]
+      if options[:show_settings]
+        show_settings(options)
+      elsif options[:filter_mode]
         execute_as_filter(options)
       else
         execute_as_converter(options)
       end
     end
 
+    def self.show_settings(options)
+      bots_re = compile_bots_re_from_config_file(options[:bots_config_file])
+      parser = choose_log_parser(options[:log_format])
+      puts "The regular expression for bots: #{bots_re}"
+      puts "LogFormat: #{parser.format_strings}"
+    end
+
     def self.execute_as_filter(options)
-      configs = load_config_file(options[:config_file])
+      configs = Utils.load_config_file(options[:config_file])
       parser = choose_log_parser(options[:log_format])
       output_dir = options[:output_dir]
-      output_log_names = collect_output_log_names(configs)
-      Utils.open_multiple_output_files(output_log_names, output_dir) do |logs|
-        queries = setup_queries_from_configs(configs, logs)
-        LogLineParser.each_record(parser: parser) do |line, record|
-          queries.each {|query| query.call(line, record) }
-        end
-      end
+      bots_re = compile_bots_re_from_config_file(options[:bots_config_file])
+      execute_queries(configs, parser, output_dir, bots_re)
     end
 
     def self.execute_as_converter(options, output=STDOUT, input=ARGF)
@@ -101,7 +106,7 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
       end
     end
 
-    private
+    # private class methods
 
     def self.predefined_options_for_log_format
       PREDEFINED_FORMATS.keys.
@@ -109,15 +114,31 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
         join(", ")
     end
 
+    def self.compile_bots_re_from_config_file(bots_config_file)
+      return Bots::DEFAULT_RE unless bots_config_file
+      configs = Utils.load_config_file(bots_config_file)[0]
+      Bots.compile_bots_re(configs)
+    end
+
     def self.collect_output_log_names(configs)
       configs.map do |config|
         config[Query::ConfigFields::OUTPUT_LOG_NAME]
       end
     end
 
-    def self.setup_queries_from_configs(configs, logs)
+    def self.execute_queries(configs, parser, output_dir, bots_re)
+      output_log_names = collect_output_log_names(configs)
+      Utils.open_multiple_output_files(output_log_names, output_dir) do |logs|
+        queries = setup_queries_from_configs(configs, logs, bots_re)
+        LogLineParser.each_record(parser: parser) do |line, record|
+          queries.each {|query| query.call(line, record) }
+        end
+      end
+    end
+
+    def self.setup_queries_from_configs(configs, logs, bots_re)
       configs.map do |config|
-        Query.register_query_to_log(config, logs)
+        Query.register_query_to_log(config, logs, bots_re)
       end
     end
 
@@ -138,5 +159,14 @@ formats predefined as #{predefined_options_for_log_format}") do |log_format|
         output.puts parser.to_ltsv(line.chomp)
       end
     end
+
+    private_class_method(:predefined_options_for_log_format,
+                         :compile_bots_re_from_config_file,
+                         :collect_output_log_names,
+                         :execute_queries,
+                         :setup_queries_from_configs,
+                         :convert_to_csv,
+                         :convert_to_tsv,
+                         :convert_to_ltsv)
   end
 end

data/lib/log_line_parser/query.rb

@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 
+require 'log_line_parser/bots'
+
 module LogLineParser
   class Query
     class NotAllowableMethodError < StandardError; end
@@ -18,20 +20,12 @@ module LogLineParser
 
     TAIL_SLASH_RE = /\/$/
     SLASH = '/'
-    DEFAULT_BOTS = %w(
-Googlebot
-Googlebot-Mobile
-Mediapartners-Google
-Bingbot
-Slurp
-Baiduspider
-BaiduImagespider
-BaiduMobaider
-YetiBot
-)
+
+    IMAGE_FILE_RE = /\.(?:jpe?g|png|gif|ico|tiff?|bmp|svgz?|webp)$/in
 
     ALLOWABLE_METHODS = [
       :access_by_bots?,
+      :access_to_image?,
       :referred_from_resources?,
       :referred_from_under_resources?,
       :access_to_resources?,
@@ -64,15 +58,12 @@ YetiBot
       MATCH_TYPE = "match_type" # The value should be "all" or "any".
     end
 
-    def self.compile_bots_re(bot_names=DEFAULT_BOTS)
-      bots_str = bot_names.map {|name| Regexp.escape(name) }.join("|")
-      Regexp.compile(bots_str, Regexp::IGNORECASE)
+    def self.access_by_bots?(record, bots_re=Bots::DEFAULT_RE)
+      bots_re =~ record.user_agent
     end
 
-    DEFAULT_BOTS_RE = compile_bots_re
-
-    def self.access_by_bots?(record, bots_re=DEFAULT_BOTS_RE)
-      bots_re =~ record.user_agent
+    def self.access_to_image?(record)
+      IMAGE_FILE_RE =~ record.resource
     end
 
     ##
@@ -95,10 +86,15 @@ YetiBot
       record.resource.start_with?(path)
     end
 
+    def self.referred_from_host?(record, host_name)
+      record.referer_host == host_name
+    end
+
     class << self
-      def register_query_to_log(option, logs)
+      def register_query_to_log(option, logs, bots_re=Bots::DEFAULT_RE)
         query = Query.new(domain: option[ConfigFields::HOST_NAME],
-                          resources: option[ConfigFields::RESOURCES])
+                          resources: option[ConfigFields::RESOURCES],
+                          bots_re: bots_re)
         queries = option[ConfigFields::MATCH]
         reject_unacceptable_queries(queries)
         log = logs[option[ConfigFields::OUTPUT_LOG_NAME]]
@@ -176,15 +172,20 @@ YetiBot
       end
     end
 
-    def initialize(domain: nil, resources: [])
+    def initialize(domain: nil, resources: [], bots_re: Bots::DEFAULT_RE)
       @domain = domain
       @resources = normalize_resources(resources)
+      @bots_re = bots_re
       @normalized_resources = normalize_resources(resources)
       @normalized_dirs = @normalized_resources - @resources
     end
 
-    def access_by_bots?(record, bots_re=DEFAULT_BOTS_RE)
-      bots_re =~ record.user_agent
+    def access_by_bots?(record)
+      @bots_re =~ record.user_agent
+    end
+
+    def access_to_image?(record)
+      IMAGE_FILE_RE =~ record.resource
     end
 
     ##

data/lib/log_line_parser/utils.rb

@@ -3,6 +3,7 @@
 require 'log_line_parser'
 require 'log_line_parser/query'
 require 'csv'
+require 'yaml'
 
 module LogLineParser
   module Utils
@@ -15,10 +16,6 @@ module LogLineParser
     }
     SPECIAL_CHARS_RE = Regexp.compile(SPECIAL_CHARS.keys.join("|"))
 
-    def self.access_by_bots?(record, bots_re=Query::DEFAULT_BOTS_RE)
-      Query.access_by_bots?(record, bots_re)
-    end
-
     def self.open_multiple_output_files(base_names, dir=nil, ext="log")
       logs = {}
       filepath = dir ? File.join(dir, "%s.#{ext}") : "%s.#{ext}"
@@ -32,6 +29,16 @@ module LogLineParser
       end
     end
 
+    def self.read_configs(config)
+      YAML.load_stream(config).to_a
+    end
+
+    def self.load_config_file(config_file)
+      open(File.expand_path(config_file)) do |f|
+        read_configs(f.read)
+      end
+    end
+
     def self.to_tsv(line, escape=true)
       LogLineParser.parse(line).to_a.map do |field|
         escape ? escape_special_chars(field) : field
@@ -42,12 +49,14 @@ module LogLineParser
       LogLineParser.parse(line).to_a.to_csv
     end
 
-    private
+    # private class methods
 
     def self.escape_special_chars(field)
       field.gsub(SPECIAL_CHARS_RE) do |char|
         SPECIAL_CHARS[char]
       end
     end
+
+    private_class_method :escape_special_chars
   end
 end

data/lib/log_line_parser/version.rb

@@ -1,3 +1,3 @@
 module LogLineParser
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/log_line_parser.rb

@@ -197,7 +197,7 @@ module LogLineParser
   #
   # For example,
   #
-  #    parser = LogLineParse.parser("%h %l %u %t \"%r\" %>s %b")
+  #    parser = LogLineParser.parser("%h %l %u %t \"%r\" %>s %b")
   #
   # creates the parser of Common Log Format.
 

data/samples/output/index-page-accessed-by-bot.log

@@ -1 +1,2 @@
 192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"

data/samples/sample_combined_log.log

@@ -10,3 +10,4 @@
 192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
 192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external2.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
 192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_line_parser
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - HASHIMOTO, Naoki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-27 00:00:00.000000000 Z
+date: 2016-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,6 +60,7 @@ files:
 - exe/log_line_parser
 - lib/log_line_parser.rb
 - lib/log_line_parser/apache.rb
+- lib/log_line_parser/bots.rb
 - lib/log_line_parser/command_line_interface.rb
 - lib/log_line_parser/line_parser.rb
 - lib/log_line_parser/ltsv.rb