log_line_parser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.travis.yml +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +216 -0
- data/Rakefile +11 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/exe/log_line_parser +7 -0
- data/lib/log_line_parser/apache.rb +80 -0
- data/lib/log_line_parser/command_line_interface.rb +126 -0
- data/lib/log_line_parser/line_parser.rb +211 -0
- data/lib/log_line_parser/moe.rb +18 -0
- data/lib/log_line_parser/query.rb +290 -0
- data/lib/log_line_parser/utils.rb +53 -0
- data/lib/log_line_parser/version.rb +3 -0
- data/lib/log_line_parser.rb +232 -0
- data/log_line_parser.gemspec +33 -0
- data/samples/output/access-to-two-specific-files.log +2 -0
- data/samples/output/all-but-bots-and-not-found.log +10 -0
- data/samples/output/all-records-related-to-subdir_index.log +4 -0
- data/samples/output/index-page-accessed-by-bot.log +1 -0
- data/samples/output/referred-from-external-site.log +1 -0
- data/samples/sample_combined_log.log +12 -0
- data/samples/sample_config.yml +46 -0
- metadata +101 -0
@@ -0,0 +1,232 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "log_line_parser/version"
|
4
|
+
require "log_line_parser/line_parser"
|
5
|
+
require "log_line_parser/apache"
|
6
|
+
require "strscan"
|
7
|
+
require "time"
|
8
|
+
require "date"
|
9
|
+
|
10
|
+
module LogLineParser
|
11
|
+
include LineParser
|
12
|
+
extend LineParser::Helpers
|
13
|
+
|
14
|
+
class MalFormedRecordError < StandardError; end
|
15
|
+
|
16
|
+
module Fields
|
17
|
+
# LogFormat "%h %l %u %t \"%r\" %>s %b" common
|
18
|
+
COMMON = [
|
19
|
+
:remote_host,
|
20
|
+
:remote_logname,
|
21
|
+
:remote_user,
|
22
|
+
:time,
|
23
|
+
:first_line_of_request,
|
24
|
+
:last_request_status,
|
25
|
+
:response_bytes,
|
26
|
+
].freeze
|
27
|
+
|
28
|
+
# LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
|
29
|
+
COMBINED = (COMMON + [:referer, :user_agent]).freeze
|
30
|
+
end
|
31
|
+
|
32
|
+
PREDEFINED_FORMATS = {}
|
33
|
+
|
34
|
+
class LogLineTokenizer < Tokenizer
|
35
|
+
setup(%w([ ] - \\ "), ['\s+']) #"
|
36
|
+
end
|
37
|
+
|
38
|
+
define_nodes(RootNode: [nil, nil, [" "]],
|
39
|
+
BasicFieldNode: [nil, " ", []],
|
40
|
+
TimeNode: ["[", "]", []],
|
41
|
+
StringNode: ['"', '"', []])
|
42
|
+
|
43
|
+
class StringEscapeNode < EscapeNode
|
44
|
+
setup('\\', nil, [], ['\\', '"', 't', 'n', 'r'])
|
45
|
+
ESCAPED = {
|
46
|
+
'\\' => '\\',
|
47
|
+
'"' => '"',
|
48
|
+
't' => "\t",
|
49
|
+
'n' => "\n",
|
50
|
+
'r' => "\r",
|
51
|
+
}
|
52
|
+
|
53
|
+
def to_s
|
54
|
+
ESCAPED[@subnodes[0]] || ''.freeze
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
define_node_nesting(RootNode => [TimeNode, StringNode],
|
59
|
+
StringNode => [StringEscapeNode])
|
60
|
+
|
61
|
+
class LogLineNodeStack < NodeStack
|
62
|
+
setup(RootNode, BasicFieldNode)
|
63
|
+
|
64
|
+
def to_a
|
65
|
+
root.subnodes.map {|node| node.to_s }
|
66
|
+
end
|
67
|
+
|
68
|
+
def to_hash(record_type=CombinedLogParser)
|
69
|
+
record_type.to_hash(to_a)
|
70
|
+
end
|
71
|
+
|
72
|
+
def to_record(record_type=CombinedLogParser)
|
73
|
+
record_type.create(to_a)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
module ClassMethods
|
78
|
+
DATE_TIME_SEP = /:/
|
79
|
+
|
80
|
+
attr_accessor :parse_time_value, :format_strings
|
81
|
+
|
82
|
+
def setup(field_names, format_strings=nil)
|
83
|
+
@field_names = field_names
|
84
|
+
@format_strings = format_strings
|
85
|
+
@number_of_fields = field_names.length
|
86
|
+
@referer_defined = field_names.include?(:referer)
|
87
|
+
@parse_time_value = false
|
88
|
+
end
|
89
|
+
|
90
|
+
def parse(line)
|
91
|
+
fields = LogLineParser.parse(line).to_a
|
92
|
+
unless fields.length == @number_of_fields
|
93
|
+
raise MalFormedRecordError, line
|
94
|
+
end
|
95
|
+
create(fields)
|
96
|
+
end
|
97
|
+
|
98
|
+
def create(log_fields)
|
99
|
+
new(*log_fields).tap do |rec|
|
100
|
+
rec.last_request_status = rec.last_request_status.to_i
|
101
|
+
rec.response_bytes = response_size(rec)
|
102
|
+
rec.time = parse_time(rec.time) if @parse_time_value
|
103
|
+
rec.parse_request
|
104
|
+
rec.parse_referer if @referer_defined
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def to_hash(line)
|
109
|
+
values = line.kind_of?(Array) ? line : LogLineParser.parse(line).to_a
|
110
|
+
h = {}
|
111
|
+
@format_strings.each_with_index do |key, i|
|
112
|
+
h[key] = values[i]
|
113
|
+
end
|
114
|
+
parse_request(h)
|
115
|
+
h
|
116
|
+
end
|
117
|
+
|
118
|
+
def parse_request(h)
|
119
|
+
if first_line_of_request = h["%r".freeze]
|
120
|
+
request = first_line_of_request.split(/ /)
|
121
|
+
h["%m"] ||= request.shift
|
122
|
+
h["%H"] ||= request.pop
|
123
|
+
h["%U%q"] ||= request.size == 1 ? request[0] : request.join(" ".freeze)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
def response_size(rec)
|
130
|
+
size_str = rec.response_bytes
|
131
|
+
size_str == "-".freeze ? 0 : size_str.to_i
|
132
|
+
end
|
133
|
+
|
134
|
+
def parse_time(time_str)
|
135
|
+
Time.parse(time_str.sub(DATE_TIME_SEP, " ".freeze))
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
module InstanceMethods
|
140
|
+
SPACE_RE = / /
|
141
|
+
SLASH_RE = /\//
|
142
|
+
SLASH = '/'.freeze
|
143
|
+
SCHEMES =%w(http: https:)
|
144
|
+
|
145
|
+
attr_reader :method, :protocol, :resource
|
146
|
+
attr_reader :referer_scheme, :referer_host, :referer_resource
|
147
|
+
|
148
|
+
def date(offset=0)
|
149
|
+
DateTime.parse((self.time + offset * 86400).to_s)
|
150
|
+
end
|
151
|
+
|
152
|
+
def parse_request
|
153
|
+
request = self.first_line_of_request.split(SPACE_RE)
|
154
|
+
@method = request.shift
|
155
|
+
@protocol = request.pop
|
156
|
+
@resource = request.size == 1 ? request[0] : request.join(" ".freeze)
|
157
|
+
end
|
158
|
+
|
159
|
+
def parse_referer
|
160
|
+
return if self.referer == "-".freeze
|
161
|
+
parts = self.referer.split(SLASH_RE, 4)
|
162
|
+
if SCHEMES.include? parts[0]
|
163
|
+
@referer_scheme = parts[0]
|
164
|
+
@referer_host = parts[2]
|
165
|
+
@referer_resource = parts[3] ? SLASH + parts[3] : SLASH
|
166
|
+
else
|
167
|
+
@referer_scheme = "".freeze
|
168
|
+
@referer_host = "".freeze
|
169
|
+
@referer_resource = self.referer
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def referred_from_host?(host_name)
|
174
|
+
@referer_host == host_name
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def self.create_record_type(field_names, format_strings)
|
179
|
+
record_type = Struct.new(*field_names)
|
180
|
+
record_type.extend(ClassMethods)
|
181
|
+
record_type.include(InstanceMethods)
|
182
|
+
record_type.setup(field_names, format_strings)
|
183
|
+
record_type
|
184
|
+
end
|
185
|
+
|
186
|
+
def self.parser(log_format)
|
187
|
+
if log_format.kind_of? String
|
188
|
+
format_strings = Apache.parse_log_format(log_format)
|
189
|
+
field_names = Apache.format_strings_to_symbols(format_strings)
|
190
|
+
else
|
191
|
+
format_strings = nil
|
192
|
+
field_names = log_format
|
193
|
+
end
|
194
|
+
|
195
|
+
create_record_type(field_names, format_strings)
|
196
|
+
end
|
197
|
+
|
198
|
+
def self.parse(line)
|
199
|
+
stack = LogLineNodeStack.new
|
200
|
+
tokens = LogLineTokenizer.tokenize(line.chomp)
|
201
|
+
tokens.each {|token| stack.push token }
|
202
|
+
stack
|
203
|
+
# I'm not checking the reason yet, but the following way of pushing
|
204
|
+
# tokens directly into the stack is very slow.
|
205
|
+
#
|
206
|
+
# LogLineTokenizer.tokenize(line.chomp, stack)
|
207
|
+
end
|
208
|
+
|
209
|
+
def self.to_array(line)
|
210
|
+
parse(line).to_a
|
211
|
+
end
|
212
|
+
|
213
|
+
CommonLogParser = parser(Apache::LogFormat::COMMON)
|
214
|
+
CommonLogWithVHParser = parser(Apache::LogFormat::COMMON_WITH_VH)
|
215
|
+
CombinedLogParser = parser(Apache::LogFormat::COMBINED)
|
216
|
+
|
217
|
+
PREDEFINED_FORMATS['common'] = CommonLogParser
|
218
|
+
PREDEFINED_FORMATS['common_with_vh'] = CommonLogWithVHParser
|
219
|
+
PREDEFINED_FORMATS['combined'] = CombinedLogParser
|
220
|
+
|
221
|
+
def self.each_record(record_type: CommonLogParser,
|
222
|
+
input: ARGF,
|
223
|
+
error_output: STDERR)
|
224
|
+
input.each_line do |line|
|
225
|
+
begin
|
226
|
+
yield line, record_type.parse(line)
|
227
|
+
rescue MalFormedRecordError => e
|
228
|
+
error_output.print e.message
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'log_line_parser/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "log_line_parser"
|
8
|
+
spec.version = LogLineParser::VERSION
|
9
|
+
spec.required_ruby_version = ">= 2.0.0"
|
10
|
+
spec.authors = ["HASHIMOTO, Naoki"]
|
11
|
+
spec.email = ["hashimoto.naoki@gmail.com"]
|
12
|
+
|
13
|
+
spec.summary = %q{A simple parser of Apache access logs}
|
14
|
+
spec.description = %q{A simple parser of Apache access logs: it parses a line of Apache access log and turns it into an array of strings or a Hash object. And from the command line, you can use it as a conversion tool of file formats or as a filtering tool of access records.}
|
15
|
+
spec.homepage = "https://github.com/nico-hn/LogLineParser"
|
16
|
+
spec.license = "MIT"
|
17
|
+
|
18
|
+
# # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
|
19
|
+
# # delete this section to allow pushing this gem to any host.
|
20
|
+
# if spec.respond_to?(:metadata)
|
21
|
+
# spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
|
22
|
+
# else
|
23
|
+
# raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
24
|
+
# end
|
25
|
+
|
26
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
27
|
+
spec.bindir = "exe"
|
28
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
29
|
+
spec.require_paths = ["lib"]
|
30
|
+
|
31
|
+
spec.add_development_dependency "bundler", "~> 1.9"
|
32
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
33
|
+
end
|
@@ -0,0 +1,2 @@
|
|
1
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /end.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
2
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
@@ -0,0 +1,10 @@
|
|
1
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
2
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /end.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
3
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
4
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir HTTP/1.1" 301 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
5
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
6
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
7
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
8
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html?q=try+to+find+something HTTP/1.1" 200 432 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
9
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
10
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external2.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
@@ -0,0 +1,4 @@
|
|
1
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
2
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
3
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
4
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html?q=try+to+find+something HTTP/1.1" 200 432 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
@@ -0,0 +1 @@
|
|
1
|
+
192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
@@ -0,0 +1 @@
|
|
1
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
@@ -0,0 +1,12 @@
|
|
1
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
2
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /end.html HTTP/1.1" 200 432 "http://www.example.org/start.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
3
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
4
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /non-existent.html HTTP/1.1" 404 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
5
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir HTTP/1.1" 301 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
6
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/big.pdf HTTP/1.1" 206 16384 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
7
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
8
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html HTTP/1.1" 200 432 "http://www.example.org/" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
9
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /subdir/index.html?q=try+to+find+something HTTP/1.1" 200 432 "http://www.example.org/subdir/index.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
10
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
11
|
+
192.168.3.4 - quidam [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.net/external2.html" "Mozilla/5.0 (X11; U; Linux i686; ja-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0"
|
12
|
+
192.168.3.4 - - [07/Feb/2016:07:39:42 +0900] "GET /index.html HTTP/1.1" 200 432 "http://www.example.org" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
@@ -0,0 +1,46 @@
|
|
1
|
+
---
|
2
|
+
host_name: www.example.org
|
3
|
+
resources:
|
4
|
+
- /subdir/index.html
|
5
|
+
match:
|
6
|
+
- :access_to_under_resources?
|
7
|
+
- :referred_from_resources?
|
8
|
+
match_type: any
|
9
|
+
output_log_name: all-records-related-to-subdir_index
|
10
|
+
---
|
11
|
+
host_name: www.example.org
|
12
|
+
resources:
|
13
|
+
- /end.html
|
14
|
+
- /subdir/big.pdf
|
15
|
+
match:
|
16
|
+
- :access_to_resources?
|
17
|
+
match_type: any
|
18
|
+
output_log_name: access-to-two-specific-files
|
19
|
+
---
|
20
|
+
host_name: www.example.org
|
21
|
+
resources:
|
22
|
+
- /
|
23
|
+
match:
|
24
|
+
- :access_to_under_resources?
|
25
|
+
match_type: any
|
26
|
+
ignore_match:
|
27
|
+
- :access_by_bots?
|
28
|
+
- :not_found?
|
29
|
+
output_log_name: all-but-bots-and-not-found
|
30
|
+
---
|
31
|
+
host_name: www.example.org
|
32
|
+
resources:
|
33
|
+
- /index.html
|
34
|
+
match:
|
35
|
+
- :access_to_resources?
|
36
|
+
- :access_by_bots?
|
37
|
+
match_type: all
|
38
|
+
output_log_name: index-page-accessed-by-bot
|
39
|
+
---
|
40
|
+
host_name: www.example.net
|
41
|
+
resources:
|
42
|
+
- /external.html
|
43
|
+
match:
|
44
|
+
- :referred_from_resources?
|
45
|
+
match_type: all
|
46
|
+
output_log_name: referred-from-external-site
|
metadata
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: log_line_parser
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- HASHIMOTO, Naoki
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-03-06 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.9'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.9'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
description: 'A simple parser of Apache access logs: it parses a line of Apache access
|
42
|
+
log and turns it into an array of strings or a Hash object. And from the command
|
43
|
+
line, you can use it as a conversion tool of file formats or as a filtering tool
|
44
|
+
of access records.'
|
45
|
+
email:
|
46
|
+
- hashimoto.naoki@gmail.com
|
47
|
+
executables:
|
48
|
+
- log_line_parser
|
49
|
+
extensions: []
|
50
|
+
extra_rdoc_files: []
|
51
|
+
files:
|
52
|
+
- ".gitignore"
|
53
|
+
- ".travis.yml"
|
54
|
+
- Gemfile
|
55
|
+
- LICENSE.txt
|
56
|
+
- README.md
|
57
|
+
- Rakefile
|
58
|
+
- bin/console
|
59
|
+
- bin/setup
|
60
|
+
- exe/log_line_parser
|
61
|
+
- lib/log_line_parser.rb
|
62
|
+
- lib/log_line_parser/apache.rb
|
63
|
+
- lib/log_line_parser/command_line_interface.rb
|
64
|
+
- lib/log_line_parser/line_parser.rb
|
65
|
+
- lib/log_line_parser/moe.rb
|
66
|
+
- lib/log_line_parser/query.rb
|
67
|
+
- lib/log_line_parser/utils.rb
|
68
|
+
- lib/log_line_parser/version.rb
|
69
|
+
- log_line_parser.gemspec
|
70
|
+
- samples/output/access-to-two-specific-files.log
|
71
|
+
- samples/output/all-but-bots-and-not-found.log
|
72
|
+
- samples/output/all-records-related-to-subdir_index.log
|
73
|
+
- samples/output/index-page-accessed-by-bot.log
|
74
|
+
- samples/output/referred-from-external-site.log
|
75
|
+
- samples/sample_combined_log.log
|
76
|
+
- samples/sample_config.yml
|
77
|
+
homepage: https://github.com/nico-hn/LogLineParser
|
78
|
+
licenses:
|
79
|
+
- MIT
|
80
|
+
metadata: {}
|
81
|
+
post_install_message:
|
82
|
+
rdoc_options: []
|
83
|
+
require_paths:
|
84
|
+
- lib
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 2.0.0
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
requirements: []
|
96
|
+
rubyforge_project:
|
97
|
+
rubygems_version: 2.2.3
|
98
|
+
signing_key:
|
99
|
+
specification_version: 4
|
100
|
+
summary: A simple parser of Apache access logs
|
101
|
+
test_files: []
|