apache_log-parser 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 338af4fdaec6a5a65bf000f5dc549f73ba1db221
-  data.tar.gz: b5366447d647a4da67dbce66468730fa993fbd15
+  metadata.gz: 2679264013d2c6cf84bb77978e6380edd1c4ab67
+  data.tar.gz: 71b9d2114b248aaae424daadcfdd5e86e1cc9528
 SHA512:
-  metadata.gz: fe677bb02e77958f076a1b68a1b6cf8f5c619c4b7f7c551e6e8ef9b7496617a13b7b25ff268881d0f6f9f3a3f978c2ee6165500a2477b507d8cb91d5eba79c13
-  data.tar.gz: 2ee1ab1f75310b24e9ad8cdcfabcaa69607e76ce83534301519cfba81640ef3ebe80be56a35b7efde54f2b71c3d599a0c0eb443c341d670769b58551dfabf48a
+  metadata.gz: 3281247ab7a73b393dea4364c6595ee9adaa3803767555e3b95ccb575d6159ed5a0b0589e8f5ac6f1b98718ed666ab66be53a67718cf75799bf28f856aa25a06
+  data.tar.gz: e50f2cf1469c4a665581e4d2fbcba1490800033bb65f9e250b705a8dba821708dba266392177b776b738664571f36a2bb719b88213d42cffb75afbd002523b9d

data/lib/apache_log/parser.rb

@@ -7,7 +7,7 @@ module ApacheLog
       common_fields   = %w(remote_host identity_check user datetime request status size)
       combined_fields = common_fields + %w(referer user_agent)
 
-      common_pattern     = '(\S+)\s+(\S+)\s+(\S+)\s+\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]\s+"(\S+\s\S+\s\S+)"\s+(\S+)\s+(\S+)'
+      common_pattern     = '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+(\S+)\s+(\S+)\s+\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]\s+"(\S+\s\S+\s\S+)"\s+(\S+)\s+(\S+)'
       combined_pattern   = common_pattern + '\s+"([^"]*)"\s+"([^"]*)"'
       additional_pattern = ''
 
@@ -18,10 +18,10 @@ module ApacheLog
       case format
       when 'common'
         @fields = common_fields + additional_fields
-        @pattern = /^#{common_pattern}#{additional_pattern}$/
+        @pattern = /#{common_pattern}#{additional_pattern}/
       when 'combined'
         @fields = combined_fields + additional_fields
-        @pattern = /^#{combined_pattern}#{additional_pattern}$/
+        @pattern = /#{combined_pattern}#{additional_pattern}/
       else
         raise "format error\n no such format: <#{format}> \n"
       end

data/lib/apache_log/parser/version.rb

@@ -1,5 +1,5 @@
 module ApacheLog
   class Parser
-    VERSION = "3.0.0"
+    VERSION = "3.1.0"
   end
 end

data/spec/apache_log/parser_spec.rb

@@ -26,10 +26,10 @@ describe ApacheLog::Parser do
   end
 
   it 'can parse combined format log' do
-    line = '104.24.160.39 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
+    line = '192.168.0.1 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
     parser = ApacheLog::Parser.new('combined')
     entity = parser.parse(line.chomp)
-    expect = {remote_host: '104.24.160.39', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
       request: {method: 'GET', path: '/category/electronics', protocol: 'HTTP/1.1'}, status: '200', size: '128', referer: '-',
       user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}
     expect(entity).to eq(expect)
@@ -46,20 +46,20 @@ describe ApacheLog::Parser do
   end
 
   it 'can parse tab separated combined format log' do
-    line = "203.0.113.254\t-\t-\t[07/Feb/2011:10:59:59 +0900]\t\"GET /x/i.cgi/movie/0001/-0002 HTTP/1.1\"\t200\t14462\t\"-\"\t\"DoCoMo/2.0 F08A3(c500;TB;W30H20)\"";
+    line = "192.168.0.1\t-\t-\t[07/Feb/2011:10:59:59 +0900]\t\"GET /x/i.cgi/movie/0001/-0002 HTTP/1.1\"\t200\t14462\t\"-\"\t\"DoCoMo/2.0 F08A3(c500;TB;W30H20)\"";
     parser = ApacheLog::Parser.new('combined')
     entity = parser.parse(line.chomp)
-    expect = {remote_host: '203.0.113.254', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
       request: {method: 'GET', path: '/x/i.cgi/movie/0001/-0002', protocol: 'HTTP/1.1'}, status: '200', size: '14462', referer: '-',
       user_agent: 'DoCoMo/2.0 F08A3(c500;TB;W30H20)'}
     expect(entity).to eq(expect)
   end
 
   it 'can parse custom format log based on combined format' do
-    line = '104.24.160.39 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1" "example.com" "192.168.0.1201102091208001" "901"'
+    line = '192.168.0.1 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1" "example.com" "192.168.0.1201102091208001" "901"'
     parser = ApacheLog::Parser.new('combined', %w(vhost usertrack request_duration))
     entity = parser.parse(line.chomp)
-    expect = {remote_host: '104.24.160.39', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
       request: {method: 'GET', path: '/category/electronics', protocol: 'HTTP/1.1'}, status: '200', size: '128', referer: '-',
       user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', vhost: 'example.com',
       usertrack: '192.168.0.1201102091208001', request_duration: '901'}
@@ -78,13 +78,51 @@ describe ApacheLog::Parser do
   end
 
   it 'can parse tab separated custom format log based on combined format' do
-    line = "203.0.113.254\t-\t-\t[07/Feb/2011:10:59:59 +0900]\t\"GET /x/i.cgi/movie/0001/-0002 HTTP/1.1\"\t200\t14462\t\"http://headlines.yahoo.co.jp/hl\"\t\"DoCoMo/2.0 F08A3(c500;TB;W30H20)\"\t\"virtualhost.example.jp\"\t\"192.0.2.16794832933550\"\t\"09011112222333_xx.ezweb.ne.jp\"\t533593";
+    line = "192.168.0.1\t-\t-\t[07/Feb/2011:10:59:59 +0900]\t\"GET /x/i.cgi/movie/0001/-0002 HTTP/1.1\"\t200\t14462\t\"http://headlines.yahoo.co.jp/hl\"\t\"DoCoMo/2.0 F08A3(c500;TB;W30H20)\"\t\"virtualhost.example.jp\"\t\"192.0.2.16794832933550\"\t\"09011112222333_xx.ezweb.ne.jp\"\t533593";
     parser = ApacheLog::Parser.new('combined', %w(vhost usertrack mobileid request_duration))
     entity = parser.parse(line.chomp)
-    expect = {remote_host: '203.0.113.254', identity_check: '-', user: '-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
       request: {method: 'GET', path: '/x/i.cgi/movie/0001/-0002', protocol: 'HTTP/1.1'}, status: '200', size: '14462', referer: 'http://headlines.yahoo.co.jp/hl',
       user_agent: 'DoCoMo/2.0 F08A3(c500;TB;W30H20)', vhost: 'virtualhost.example.jp',
       usertrack: '192.0.2.16794832933550', mobileid: '09011112222333_xx.ezweb.ne.jp', request_duration: '533593'}
     expect(entity).to eq(expect)
   end
+
+  it 'can parse even if there are any columns at the beginning of line' do
+    line = 'foo 127.0.0.1 - - [20/May/2014:20:04:04 +0900] "GET /test/indx.html HTTP/1.1" 200 4576'
+    parser = ApacheLog::Parser.new('common')
+    entity = parser.parse(line.chomp)
+    expect = {remote_host: '127.0.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2014, 5, 20, 20, 04, 04, 0.375),
+      request: {method: 'GET', path: '/test/indx.html', protocol: 'HTTP/1.1'}, status: '200', size: '4576'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse even if there are any columns at the beginning of line' do
+    line = '200 127.0.0.1 - - [20/May/2014:20:04:04 +0900] "GET /test/indx.html HTTP/1.1" 200 4576'
+    parser = ApacheLog::Parser.new('common')
+    entity = parser.parse(line.chomp)
+    expect = {remote_host: '127.0.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2014, 5, 20, 20, 04, 04, 0.375),
+      request: {method: 'GET', path: '/test/indx.html', protocol: 'HTTP/1.1'}, status: '200', size: '4576'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse even if there are any columns at the beginning of line' do
+    line = '200 foo 192.168.0.1 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
+    parser = ApacheLog::Parser.new('combined')
+    entity = parser.parse(line.chomp)
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
+      request: {method: 'GET', path: '/category/electronics', protocol: 'HTTP/1.1'}, status: '200', size: '128', referer: '-',
+      user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse even if there are any columns at the end of line' do
+    line = '192.168.0.1 - - [07/Jun/2014:14:58:55 +0900] "GET /category/electronics HTTP/1.1" 200 128 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1"'
+    parser = ApacheLog::Parser.new('combined')
+    entity = parser.parse(line.chomp)
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user: '-', datetime: DateTime.new(2014, 6, 7, 14, 58, 55, 0.375),
+      request: {method: 'GET', path: '/category/electronics', protocol: 'HTTP/1.1'}, status: '200', size: '128', referer: '-',
+      user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'}
+    expect(entity).to eq(expect)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apache_log-parser
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Yuichi Takada
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-11 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler