apache_log-parser 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 909728a9cf9d9f8fe6f3a5dc600da0f69a964759
-  data.tar.gz: 16d92e4ce7963c67a22cd92ead416d59e65ff3b4
+  metadata.gz: f3c8a20f2a6c50f76a758f8e44f8da1e56e8b531
+  data.tar.gz: 49e38fea3f0fb27216ed3c32f0fb40e2aed93662
 SHA512:
-  metadata.gz: 947dfe5dd1af740c8ed4ab20aeaca12ce547e4084bca890280a8ad90a209613761d25263fc13bd96648d371ef2743495c72a5b6f882cc5b184a529d0a76538de
-  data.tar.gz: 78cf579ef4fd553a0d6a6d6252af4c1511df5acd9871cda9f281c833445717a66f41e4a90765788a7741d918b9437ed5de70279a07bb5e3e727574e9b3ec6fec
+  metadata.gz: febb514e9f8f95b380be58d6b7eaf7dfe3b9b164e61aae1bd289f9e3745f27ac0748cb9d901318d1aea3524d6bbad7c258eae3e00001b424d8e9275d1da8bf35
+  data.tar.gz: a8e2766fd986b0af077bd89577649818e9e2f667d2e5d44126b5a3b81e694be9f4ca3cdb452603c4f731bf9565176aa166c52de2a898667471e58479dc34d044

data/.gitignore

@@ -21,4 +21,5 @@ tmp
 *.a
 mkmf.log
 bin/
+vendor/
 apache_log-parser-*

data/README.md

@@ -1,6 +1,6 @@
 # ApacheLog::Parser
 
-Gem to parse popular format apache log files.
+Gem to parse apache log including common, combined and customized format.
 
 ## Installation
 
@@ -21,12 +21,24 @@ Or install it yourself as:
 ```ruby
 require 'apache_log/parser'
 
-parser = ApacheLog::Parser.getParser(format)
-entity = []
-
-File.foreach(logfile) do |line|
-  entity << parser.parse(line.chomp)
-end
+# common format
+common_log = ApacheLog::Parser.parse(log_line, 'common')
+common_log[:remote_host]    #=> remote host
+common_log[:datetime]       #=> datetime
+common_log[:request]        #=> request
+
+# combined format
+common_log = ApacheLog::Parser.parse(log_line, 'combined')
+common_log[:referer]        #=> remote host
+common_log[:user_agent]     #=> datetime
+
+# custom format(additional fields after 'combined')
+# custom format: LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%v\" \"%{cookie}n\" %D"
+common_log = ApacheLog::Parser.parse(log_line, 'combined', %w(vhost usertrack request_duration))
+common_log[:user_agent]        #=> datetime
+common_log[:vhost]             #=> vhost
+common_log[:usertrack]         #=> usertrack
+common_log[:request_duration]  #=> request_duration
 ```
 
 The format parameter must be 'common' or 'combined'.

data/Rakefile

@@ -1,2 +1,6 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
 
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/apache_log-parser.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = ApacheLog::Parser::VERSION
   spec.authors       = ["Yuichi Takada"]
   spec.email         = ["takadyy@gmail.com"]
-  spec.summary       = "Gem to parse popular format apache log files."
-  spec.description   = "You can parse common or combined format log."
+  spec.summary       = "Gem to parse apache log including common, combined and customized format."
+  spec.description   = "You can parse common, combined and customized format apache log."
   spec.homepage      = "https://github.com/takady/apache_log-parser"
   spec.license       = "MIT"
 
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
 end

data/lib/apache_log/parser.rb

@@ -1,18 +1,65 @@
 require "apache_log/parser/version"
-require "apache_log/parser/common"
-require "apache_log/parser/combined"
 
 module ApacheLog
   module Parser
-    def self.getParser(format)
+
+    def self.parse(line, format, additional_fields=[])
+
+      common_fields   = %w(remote_host identity_check user datetime request status size)
+      combined_fields = common_fields + %w(referer user_agent)
+
+      common_pattern     = '(\S+)\s+(\S+)\s+(\S+)\s+\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]\s+"(\S+\s\S+\s\S+)"\s+(\S+)\s+(\S+)'
+      combined_pattern   = common_pattern + '\s+"([^"]*)"\s+"([^"]*)"'
+      additional_pattern = ''
+
+      additional_fields.each do
+        additional_pattern += '\s+"?([^"]*)"?'
+      end
+
       case format
       when 'common'
-        ApacheLog::Parser::Common
+        fields = common_fields + additional_fields
+        pattern = /^#{common_pattern}#{additional_pattern}$/
       when 'combined'
-        ApacheLog::Parser::Combined
+        fields = combined_fields + additional_fields
+        pattern = /^#{combined_pattern}#{additional_pattern}$/
       else
         raise "format error\n no such format: <#{format}> \n"
       end
+
+      match = pattern.match(line)
+      raise "parse error\n at line: <#{line}> \n" if match.nil?
+
+      columns = match.to_a
+
+      parsed_hash = {}
+      fields.each.with_index do |val, idx|
+        val = val.to_sym
+        if val == :datetime
+          parsed_hash[val] = to_datetime(columns[idx+1])
+        elsif val == :request
+          parsed_hash[val] = parse_request(columns[idx+1])
+        else
+          parsed_hash[val] = columns[idx+1]
+        end
+      end
+
+      parsed_hash
     end
+
+    private
+      def self.to_datetime(str)
+        DateTime.strptime( str, '%d/%b/%Y:%T %z')
+      end
+
+      def self.parse_request(str)
+        method, path, protocol = str.split
+        {
+          method:   method,
+          path:     path,
+          protocol: protocol,
+        }
+      end
+
   end
 end

data/lib/apache_log/parser/version.rb

@@ -1,5 +1,5 @@
 module ApacheLog
   module Parser
-    VERSION = "1.0.0"
+    VERSION = "2.0.0"
   end
 end

data/spec/apache_log/parser_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+require 'date'
+
+describe ApacheLog::Parser do
+
+  before { @parser = ApacheLog::Parser }
+
+  it 'has a version number' do
+    expect(@parser::VERSION).not_to be nil
+  end
+
+  it 'can parse common format log' do
+    line = '127.0.0.1 - - [20/May/2014:20:04:04 +0900] "GET /test/indx.html HTTP/1.1" 200 4576'
+    entity = @parser.parse(line.chomp, 'common')
+    expect = {remote_host: '127.0.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2014, 5, 20, 20, 04, 04, 0.375),
+      request: {method: 'GET', path: '/test/indx.html', protocol: 'HTTP/1.1'}, status: '200', size: '4576'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse tab separated common format log' do
+    line = "192.168.0.1\t-\t-\t[07/Feb/2011:10:59:59 +0900]\t\"GET /x/i.cgi/net/0000/ HTTP/1.1\"\t200\t9891";
+    entity = @parser.parse(line.chomp, 'common')
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2011, 2, 07, 10, 59, 59, 0.375),
+      request: {method: 'GET', path: '/x/i.cgi/net/0000/', protocol: 'HTTP/1.1'}, status: '200', size: '9891'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse combined format log' do
+    line = '104.24.160.39 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
+    entity = @parser.parse(line.chomp, 'combined')
+    expect = {remote_host: '104.24.160.39', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
+      request: {method: 'GET', path: '/category/electronics', protocol: 'HTTP/1.1'}, status: '200', size: '128', referer: '-',
+      user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse combined format log' do
+    line = '192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891 "-" "DoCoMo/2.0 P03B(c500;TB;W24H16)"';
+    entity = @parser.parse(line.chomp, 'combined')
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+      request: {method: 'GET', path: '/x/i.cgi/net/0000/', protocol: 'HTTP/1.1'}, status: '200', size: '9891', referer: '-',
+      user_agent: 'DoCoMo/2.0 P03B(c500;TB;W24H16)'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse tab separated combined format log' do
+    line = "203.0.113.254\t-\t-\t[07/Feb/2011:10:59:59 +0900]\t\"GET /x/i.cgi/movie/0001/-0002 HTTP/1.1\"\t200\t14462\t\"-\"\t\"DoCoMo/2.0 F08A3(c500;TB;W30H20)\"";
+    entity = @parser.parse(line.chomp, 'combined')
+    expect = {remote_host: '203.0.113.254', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+      request: {method: 'GET', path: '/x/i.cgi/movie/0001/-0002', protocol: 'HTTP/1.1'}, status: '200', size: '14462', referer: '-',
+      user_agent: 'DoCoMo/2.0 F08A3(c500;TB;W30H20)'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse custom format log based on combined format' do
+    line = '104.24.160.39 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1" "example.com" "192.168.0.1201102091208001" "901"'
+    entity = @parser.parse(line.chomp, 'combined', %w(vhost usertrack request_duration))
+    expect = {remote_host: '104.24.160.39', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
+      request: {method: 'GET', path: '/category/electronics', protocol: 'HTTP/1.1'}, status: '200', size: '128', referer: '-',
+      user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', vhost: 'example.com',
+      usertrack: '192.168.0.1201102091208001', request_duration: '901'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse custom format log based on combined format' do
+    line = '192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891 "-" "DoCoMo/2.0 P03B(c500;TB;W24H16)" virtualhost.example.jp "192.0.2.16794832933550" "09011112222333_xx.ezweb.ne.jp" 533593';
+    entity = @parser.parse(line.chomp, 'combined', %w(vhost usertrack mobileid request_duration))
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+      request: {method: 'GET', path: '/x/i.cgi/net/0000/', protocol: 'HTTP/1.1'}, status: '200', size: '9891', referer: '-',
+      user_agent: 'DoCoMo/2.0 P03B(c500;TB;W24H16)', vhost: 'virtualhost.example.jp',
+      usertrack: '192.0.2.16794832933550', mobileid: '09011112222333_xx.ezweb.ne.jp', request_duration: '533593'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse tab separated custom format log based on combined format' do
+    line = "203.0.113.254\t-\t-\t[07/Feb/2011:10:59:59 +0900]\t\"GET /x/i.cgi/movie/0001/-0002 HTTP/1.1\"\t200\t14462\t\"http://headlines.yahoo.co.jp/hl\"\t\"DoCoMo/2.0 F08A3(c500;TB;W30H20)\"\t\"virtualhost.example.jp\"\t\"192.0.2.16794832933550\"\t\"09011112222333_xx.ezweb.ne.jp\"\t533593";
+    entity = @parser.parse(line.chomp, 'combined', %w(vhost usertrack mobileid request_duration))
+    expect = {remote_host: '203.0.113.254', identity_check: '-', user: '-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+      request: {method: 'GET', path: '/x/i.cgi/movie/0001/-0002', protocol: 'HTTP/1.1'}, status: '200', size: '14462', referer: 'http://headlines.yahoo.co.jp/hl',
+      user_agent: 'DoCoMo/2.0 F08A3(c500;TB;W30H20)', vhost: 'virtualhost.example.jp',
+      usertrack: '192.0.2.16794832933550', mobileid: '09011112222333_xx.ezweb.ne.jp', request_duration: '533593'}
+    expect(entity).to eq(expect)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'apache_log/parser'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apache_log-parser
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Yuichi Takada
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-14 00:00:00.000000000 Z
+date: 2014-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,7 +38,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: You can parse common or combined format log.
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: You can parse common, combined and customized format apache log.
 email:
 - takadyy@gmail.com
 executables: []
@@ -52,10 +66,9 @@ files:
 - Rakefile
 - apache_log-parser.gemspec
 - lib/apache_log/parser.rb
-- lib/apache_log/parser/combined.rb
-- lib/apache_log/parser/common.rb
-- lib/apache_log/parser/format.rb
 - lib/apache_log/parser/version.rb
+- spec/apache_log/parser_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/takady/apache_log-parser
 licenses:
 - MIT
@@ -79,5 +92,7 @@ rubyforge_project:
 rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
-summary: Gem to parse popular format apache log files.
-test_files: []
+summary: Gem to parse apache log including common, combined and customized format.
+test_files:
+- spec/apache_log/parser_spec.rb
+- spec/spec_helper.rb

data/lib/apache_log/parser/combined.rb

@@ -1,53 +0,0 @@
-require "apache_log/parser/format"
-
-module ApacheLog
-  module Parser
-    class Combined < Format
-      def initialize
-      end
-
-      def self.parse(line)
-        match = log_pattern.match(line)
-        raise "parse error\n at line: <#{line}> \n" if match.nil?
-
-        columns = match.to_a.values_at(1..9)
-        {
-          remote_host:    columns[0],
-          identity_check: columns[1],
-          user:           columns[2],
-          datetime:       to_datetime(columns[3]),
-          request:        parse_request(columns[4]),
-          status:         columns[5],
-          size:           columns[6],
-          referer:        columns[7],
-          user_agent:     columns[8],
-        }
-      end
-
-      def self.log_pattern
-        /^
-          (\S+)                                     # remote_host
-          \s+
-          (\S+)                                  # identity_check
-          \s+
-          (\S+)                                            # user
-          \s+
-          \[ (\d{2}\/.*?\d{4}:\d{2}:\d{2}:\d{2}\s.*?) \]   # date
-          \s+
-          " (.*?\s.*?\s.*?) "                           # request
-          \s+
-          (\S+)                                          # status
-          \s+
-          (\S+)                                            # size
-          \s+
-          " (.*?) "                                     # referer
-          \s+
-          " (.*?) "                                  # user_agent
-        $/x
-      end
-
-      private_class_method :log_pattern
-    end
-
-  end
-end

data/lib/apache_log/parser/common.rb

@@ -1,47 +0,0 @@
-require "apache_log/parser/format"
-
-module ApacheLog
-  module Parser
-    class Common < Format
-      def initialize
-      end
-
-      def self.parse(line)
-        match = log_pattern.match(line)
-        raise "parse error\n at line: <#{line}> \n" if match.nil?
-
-        columns = match.to_a.values_at(1..7)
-        {
-          remote_host:    columns[0],
-          identity_check: columns[1],
-          user:           columns[2],
-          datetime:       to_datetime(columns[3]),
-          request:        parse_request(columns[4]),
-          status:         columns[5],
-          size:           columns[6],
-        }
-      end
-
-      def self.log_pattern
-        /^
-          (\S+)                                     # remote_host
-          \s+
-          (\S+)                                  # identity_check
-          \s+
-          (\S+)                                            # user
-          \s+
-          \[ (\d{2}\/.*?\d{4}:\d{2}:\d{2}:\d{2}\s.*?) \]   # date
-          \s+
-          " (.*?\s.*?\s.*?) "                           # request
-          \s+
-          (\S+)                                          # status
-          \s+
-          (\S+)                                            # size
-        $/x
-      end
-
-      private_class_method :log_pattern
-    end
-
-  end
-end

data/lib/apache_log/parser/format.rb

@@ -1,32 +0,0 @@
-require 'date'
-
-module ApacheLog
-  module Parser
-    class Format
-      def initialize
-      end
-
-      def self.parse(line)
-      end
-
-      def self.log_pattern
-      end
-
-      def self.to_datetime(str)
-        DateTime.strptime( str, '%d/%b/%Y:%T %z')
-      end
-
-      def self.parse_request(str)
-        method, path, protocol = str.split
-        {
-          method:   method,
-          path:     path,
-          protocol: protocol,
-        }
-      end
-
-      private_class_method :to_datetime, :parse_request
-    end
-
-  end
-end