splog 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +4 -0
- data/lib/splog/version.rb +1 -1
- data/lib/splog-prof.rb +12 -0
- data/lib/splog.rb +82 -12
- data/splog.gemspec +1 -0
- data/test/examples/jboss/multiline_match_unmatch_server.log +1 -0
- data/test/splog_spec.rb +2 -3
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
ZTg2MDA1ZDg3MTk4NTgxOTljM2JmZjNkYTNkYzRiNDgzZjIxZDk4Yg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
ZjU1MmJiOGQxMjU0MzhmYWYyYTBkMjFiZjA5YmJjNDg4MjQxMzAxYw==
|
7
7
|
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MDg1Mjg0ZjRlYzk5NDg5YjMzMDE0ZDFhMWRiN2I4NzU5ZDc5MGIwOGVkODc0
|
10
|
+
YzgyZDg3YzhlZjJhMDhlM2UzMGY5ODIxNWZlZTZjNGI3YzI3ZWYyOWRhOThj
|
11
|
+
Y2I5ZDlmNDY4OTBkMjdkMDQ0N2RkMmE1NGJjNzdiZmM0ZTE0OTI=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
MmY4MzhjNjQwOGM1MjUzNjQ5ODg3YzI2NDQxOTA4ODJjYWQ1ZGU4NDM4NTU3
|
14
|
+
OTJiNjhlNjhlYjkzYjRjNmZkZmQ2MjUyYjdkMzg1OWQ4OWJlYzA5YjRhYzY1
|
15
|
+
MWVhMTA5OTQ0ZWY0YjcwY2M4YTU2MWM0ZjIwYzAzNGQ3ZmRkM2U=
|
data/README.md
CHANGED
@@ -126,6 +126,10 @@ Persisting the log to mongo. Set -o with no arg so that no output to stdout. T
|
|
126
126
|
|
127
127
|
bundle exec rspec test
|
128
128
|
|
129
|
+
#### Profiling in dev
|
130
|
+
|
131
|
+
head -n 10000 path_to/server.log | ruby lib/splog-prof.rb -p apache_common -o -v
|
132
|
+
|
129
133
|
#### A few performance measurements
|
130
134
|
|
131
135
|
Note that everything is done in a streaming manner with Ruby Enumerators. No file is ever read completely. At most a file is read two lines at a time. The memory requirements should be minimal due to this.
|
data/lib/splog/version.rb
CHANGED
data/lib/splog-prof.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'splog'
|
2
|
+
require 'ruby-prof'
|
3
|
+
|
4
|
+
# Profile the code
|
5
|
+
RubyProf.start
|
6
|
+
parser = Splog::LogParser.new
|
7
|
+
parser.cli(ARGV)
|
8
|
+
result = RubyProf.stop
|
9
|
+
|
10
|
+
# Print a flat profile to text
|
11
|
+
printer = RubyProf::FlatPrinter.new(result)
|
12
|
+
printer.print(STDOUT)
|
data/lib/splog.rb
CHANGED
@@ -8,6 +8,7 @@ require 'json'
|
|
8
8
|
require 'enumerator'
|
9
9
|
require 'mongo'
|
10
10
|
require 'ruby-progressbar'
|
11
|
+
require 'open3'
|
11
12
|
|
12
13
|
include Mongo
|
13
14
|
|
@@ -127,13 +128,43 @@ module Splog
|
|
127
128
|
def set_pattern(options)
|
128
129
|
@pattern_name = options[:pattern_name]
|
129
130
|
begin
|
131
|
+
# MULTILINE to match the \n chars
|
132
|
+
#Regexp::MULTILINE | Regexp::IGNORECASE
|
130
133
|
@pattern = @config[options[:pattern_name]]['regex']
|
134
|
+
delim = @config[@pattern_name].nil? ? "\\s+" : @config[@pattern_name]['delim']
|
135
|
+
c = "#{@pattern.join(delim)}"
|
136
|
+
# Remove the grouped named
|
137
|
+
@pattern_egrep = 'egrep "' + c.gsub(/\?<.*?>/, '') + '"'
|
138
|
+
r = Regexp.new(c, Regexp::MULTILINE)
|
139
|
+
@pattern = r
|
131
140
|
rescue => detail
|
141
|
+
#detail.backtrace.each { |e| $stderr.puts e}
|
142
|
+
#$stderr.puts $!
|
132
143
|
puts "No pattern matching '#{options[:pattern_name]}' found. Please choose another name or define this pattern in the your .splog.yaml"
|
133
144
|
exit
|
134
145
|
end
|
135
146
|
end
|
136
147
|
|
148
|
+
def set_pattern_match_forward
|
149
|
+
begin
|
150
|
+
@pattern_match_forward = @config[options[:pattern_name]]['match_forward_regex']
|
151
|
+
# since this is optional only compile if set
|
152
|
+
if @pattern_match_forward
|
153
|
+
delim = @config[@pattern_name].nil? ? "\\s+" : @config[@pattern_name]['delim']
|
154
|
+
# Remove the grouped named
|
155
|
+
c = "#{@pattern_match_forward.join(delim)}"
|
156
|
+
r = Regexp.new(c, Regexp::MULTILINE)
|
157
|
+
@pattern_match_forward_egrep = 'egrep "' + c.gsub(/\?<.*?>/, '') + '"'
|
158
|
+
@pattern_match_forward = r
|
159
|
+
end
|
160
|
+
rescue => detail
|
161
|
+
#detail.backtrace.each { |e| $stderr.puts e}
|
162
|
+
$stderr.puts $!
|
163
|
+
#puts "No pattern matching '#{options[:pattern_name]}' found. Please choose another name or define this pattern in the your .splog.yaml"
|
164
|
+
exit
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
137
168
|
def set_mapping(options)
|
138
169
|
begin
|
139
170
|
tmp = {}
|
@@ -183,14 +214,17 @@ module Splog
|
|
183
214
|
|
184
215
|
def parse_line(line, opts={})
|
185
216
|
res = {}
|
186
|
-
parts = opts[:parts] || @config[@pattern_name]['regex']
|
217
|
+
#parts = opts[:parts] || @config[@pattern_name]['regex']
|
187
218
|
begin
|
188
|
-
#pattern =
|
189
|
-
|
190
|
-
#
|
191
|
-
|
192
|
-
|
193
|
-
|
219
|
+
#pattern = @config[@pattern_name].has_key?('delim') ? "\\s*#{parts.join(@config[@pattern_name]['delim'])}\\s*" : "\\s*#{parts.join()}\\s*" # was working line
|
220
|
+
#r = Regexp.new(pattern, Regexp::MULTILINE)
|
221
|
+
#m = r.match(line)
|
222
|
+
|
223
|
+
if opts[:regex]
|
224
|
+
m = opts[:regex].match(line)
|
225
|
+
else
|
226
|
+
m = @pattern.match(line)
|
227
|
+
end
|
194
228
|
res = {}
|
195
229
|
if m
|
196
230
|
m.names.each do |group_name|
|
@@ -239,17 +273,40 @@ module Splog
|
|
239
273
|
parsed_line = nil
|
240
274
|
begin
|
241
275
|
while enum_ref
|
276
|
+
|
242
277
|
line = enum_ref.next
|
243
278
|
parsed_line = parse_line(line)
|
244
279
|
|
245
280
|
next_line = enum_ref.peek
|
246
281
|
# Pass in the 'match_forward_regex' if it exists so the next line can be evaluated in this context
|
247
|
-
parsed_next_line = @
|
282
|
+
#parsed_next_line = @pattern_match_forward.nil? ? parse_line(next_line) : parse_line(next_line, {:regex => @pattern_match_forward})
|
283
|
+
#parsed_next_line_test = @pattern_match_forward.nil? ? parse_line(next_line) : parse_line(next_line, {:regex => @pattern_match_forward})
|
284
|
+
|
285
|
+
# Performance optimization here, don't do a full #match only =~ since not all next lines need to be parsed period
|
286
|
+
#parsed_next_line_test = @pattern_match_forward.nil? ? next_line =~ @pattern : next_line =~ @pattern_match_forward
|
287
|
+
#egrep = "echo \"#{next_line}\" | egrep \"#{@pattern_egrep}\""
|
288
|
+
#egrep_fwd = 'echo ' + next_line + ' | egrep ' + @pattern_match_forward_egrep
|
289
|
+
#p egrep
|
290
|
+
#p egrep_fwd
|
291
|
+
#parsed_next_line_test = @pattern_match_forward.nil? ? `#{egrep}` : `#{egrep_fwd}`
|
292
|
+
|
293
|
+
o, e, s = nil
|
294
|
+
begin
|
295
|
+
o, e, s = Open3.capture3(@pattern_match_forward.nil? ? @pattern_egrep : @pattern_match_forward_egrep, :stdin_data=>next_line)
|
296
|
+
rescue Errno::EPIPE
|
297
|
+
#puts "Connection broke!"
|
298
|
+
nil
|
299
|
+
end
|
248
300
|
|
249
301
|
############################################################################################################
|
250
302
|
# If the next line matches the match_forward_regex
|
251
303
|
############################################################################################################
|
252
|
-
if parsed_next_line and @config[@pattern_name]['match_forward_regex']
|
304
|
+
#if parsed_next_line and @config[@pattern_name]['match_forward_regex']
|
305
|
+
#if not parsed_next_line_test.nil? and @config[@pattern_name]['match_forward_regex']
|
306
|
+
if s && s.success? and @config[@pattern_name]['match_forward_regex']
|
307
|
+
|
308
|
+
# Do the actual match now that we know it matches
|
309
|
+
parsed_next_line = @pattern_match_forward.nil? ? parse_line(next_line) : parse_line(next_line, {:regex => @pattern_match_forward})
|
253
310
|
|
254
311
|
# If the current_working_line does not yet exist, set it to the latest parsed line
|
255
312
|
if current_working_line.nil? and parsed_line
|
@@ -266,7 +323,8 @@ module Splog
|
|
266
323
|
while true
|
267
324
|
# Only peek here to not advance the enum unnecessarily
|
268
325
|
sub_line = enum_ref.peek
|
269
|
-
parsed_sub_line = @config[@pattern_name]['match_forward_regex'].nil? ? nil : parse_line(sub_line, {:
|
326
|
+
#parsed_sub_line = @config[@pattern_name]['match_forward_regex'].nil? ? nil : parse_line(sub_line, {:regex => @pattern_match_forward})
|
327
|
+
parsed_sub_line = @pattern_match_forward.nil? ? nil : parse_line(sub_line, {:regex => @pattern_match_forward})
|
270
328
|
if parsed_sub_line
|
271
329
|
# if matched advance the enum and add the data to the current working line
|
272
330
|
enum_ref.next
|
@@ -293,8 +351,19 @@ module Splog
|
|
293
351
|
while true
|
294
352
|
# Only peek here to not advance the enum unnecessarily
|
295
353
|
sub_line = enum_ref.peek
|
296
|
-
|
297
|
-
|
354
|
+
|
355
|
+
# TODO this can be optimized too since I'm attmpting to not match it! I don't even read the parsed_sub_line
|
356
|
+
#parsed_sub_line = parse_line(sub_line)
|
357
|
+
o, e, s = nil
|
358
|
+
begin
|
359
|
+
o, e, s = Open3.capture3(@pattern_egrep, :stdin_data=>sub_line)
|
360
|
+
rescue Errno::EPIPE
|
361
|
+
#puts "Connection broke!"
|
362
|
+
nil
|
363
|
+
end
|
364
|
+
|
365
|
+
#if parsed_sub_line.nil? and @config[@pattern_name]['unmatched_append_key_name']
|
366
|
+
if (s.nil? or not s.success?) && @config[@pattern_name]['unmatched_append_key_name']
|
298
367
|
# if unmatched advance the enum and add the data to the current working line
|
299
368
|
enum_ref.next
|
300
369
|
current_working_line[@config[@pattern_name]['unmatched_append_key_name']] << sub_line
|
@@ -446,6 +515,7 @@ module Splog
|
|
446
515
|
load_dot_file
|
447
516
|
|
448
517
|
set_pattern(options)
|
518
|
+
set_pattern_match_forward
|
449
519
|
set_mapping(options)
|
450
520
|
|
451
521
|
# Total line count, if file input we can easily do wc -l on the file. If $stdin we can allow allow a user defined
|
data/splog.gemspec
CHANGED
@@ -7,3 +7,4 @@ unmatched line 2 added to previous
|
|
7
7
|
03 Oct 2013 20:16:55,308 ERROR [stderr] (MSC service thread 1-3) Should be parsed log line # 3
|
8
8
|
03 Oct 2013 20:16:55,308 ERROR [stderr] (MSC service thread 1-3) Should be parsed log line # 4
|
9
9
|
|
10
|
+
03 Oct 2013 20:16:55,308 ERROR [stderr] (MSC service thread 1-3) Should be parsed log line # 5
|
data/test/splog_spec.rb
CHANGED
@@ -201,7 +201,7 @@ describe Splog::LogParser do
|
|
201
201
|
log_entry_two['Category'].should eql('org.jboss.as.connector.subsystems.datasources')
|
202
202
|
log_entry_two['Date'].to_s.should eql('2013-10-03 18:33:00 UTC')
|
203
203
|
log_entry_two['Date'].should be_a(Time)
|
204
|
-
log_entry_two['Message'].should eql("JBAS010403: Deploying JDBC-compliant driver class org.h2.Driver (version 1.3)\n
|
204
|
+
log_entry_two['Message'].should eql("JBAS010403: Deploying JDBC-compliant driver class org.h2.Driver (version 1.3)\n")
|
205
205
|
log_entry_two['Priority'].should eql('INFO')
|
206
206
|
log_entry_two['Thread'].should eql('ServerService Thread Pool -- 57')
|
207
207
|
end
|
@@ -219,10 +219,9 @@ describe Splog::LogParser do
|
|
219
219
|
# Get an enumerable from the parser
|
220
220
|
pe = parser.parse(e)
|
221
221
|
parsed_lines = pe.to_a
|
222
|
-
parsed_lines.length.should eql(
|
222
|
+
parsed_lines.length.should eql(5)
|
223
223
|
end
|
224
224
|
|
225
|
-
|
226
225
|
it 'should properly hash the 50 lines in the sample access log' do
|
227
226
|
# Match subsequent lines and add them to a previous line
|
228
227
|
test_dir = Dir.pwd.match(/.*?splog$/) ? 'test/' : ''
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: splog
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Samuel Mendenhall
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-10-
|
11
|
+
date: 2013-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mongo
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ~>
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '2.6'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: ruby-prof
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ! '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ! '>='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description: Parse any log file with yml defined regex rules
|
98
112
|
email:
|
99
113
|
- Samuel.Mendenhall@gmail.com
|
@@ -110,6 +124,7 @@ files:
|
|
110
124
|
- bin/splog
|
111
125
|
- examples/.splog.yml
|
112
126
|
- examples/access_log
|
127
|
+
- lib/splog-prof.rb
|
113
128
|
- lib/splog.rb
|
114
129
|
- lib/splog/version.rb
|
115
130
|
- splog.gemspec
|