winsome_wolverine 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.textile +84 -0
- data/lib/winsome_wolverine.rb +1 -0
- data/lib/wolverine.rb +23 -0
- data/lib/wolverine/Filter.rb +11 -0
- data/lib/wolverine/active_record_sink.rb +32 -0
- data/lib/wolverine/active_record_source.rb +71 -0
- data/lib/wolverine/append_indented_filter.rb +31 -0
- data/lib/wolverine/array_source.rb +10 -0
- data/lib/wolverine/core_ext.rb +9 -0
- data/lib/wolverine/count_sink.rb +12 -0
- data/lib/wolverine/enumerable.rb +27 -0
- data/lib/wolverine/event.rb +12 -0
- data/lib/wolverine/field_filter.rb +34 -0
- data/lib/wolverine/file_sink.rb +13 -0
- data/lib/wolverine/file_source.rb +38 -0
- data/lib/wolverine/filter_dsl.rb +37 -0
- data/lib/wolverine/group_filter.rb +45 -0
- data/lib/wolverine/gzip_file_source.rb +34 -0
- data/lib/wolverine/head_filter.rb +29 -0
- data/lib/wolverine/interleave_filter.rb +20 -0
- data/lib/wolverine/less_sink.rb +14 -0
- data/lib/wolverine/parse_time_filter.rb +41 -0
- data/lib/wolverine/sink.rb +11 -0
- data/lib/wolverine/source.rb +8 -0
- data/lib/wolverine/where_filter.rb +50 -0
- metadata +91 -0
data/README.textile
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
h1. Wolverine
|
2
|
+
|
3
|
+
An appetite for logs.
|
4
|
+
|
5
|
+
|
6
|
+
h2. Summary
|
7
|
+
|
8
|
+
Wolverine is a library for processing event streams. Processing starts with a source of events, such as a log file. Filters wrap a source and themselves look like a source. The DSL supplies shortcut methods to easily build a pipeline of filters in this way. Filters are processed incrementally as events are required; one can quickly get the first few events from a very long stream.
|
9
|
+
|
10
|
+
Events may be stored in a database for easy indexing. See the @ActiveRecordSource@, which provides an optimized @where@ filter.
|
11
|
+
|
12
|
+
For a list of convenience methods for building filter pipelines, see @FilterDSL@.
|
13
|
+
|
14
|
+
When processing a Rails log with multiple lines per requests and multiple web processes writing to the log concurrently, Wolverine can filter requests matching a particular criteria in a particular context. For example, it can tell you the number of exceptions thrown on POST requests, even though those bits of information are contained in different log messages.
|
15
|
+
|
16
|
+
|
17
|
+
h2. Examples
|
18
|
+
|
19
|
+
<pre>
|
20
|
+
<code>
|
21
|
+
include Wolverine
|
22
|
+
|
23
|
+
# Read events from a file, one per line
|
24
|
+
requests = FileSource.new("log/production.log").
|
25
|
+
|
26
|
+
# In this log, subsequent lines of a message may appear indented,
|
27
|
+
# so coalesce them into one event.
|
28
|
+
append_indented.
|
29
|
+
|
30
|
+
# Parse text out into fields using regex captures. Imagine a log
|
31
|
+
# line that looks like this:
|
32
|
+
# Nov 19 19:08:27.931605 dhcp-172-25-211-77 45471: Hello World
|
33
|
+
field(/\A(\w+ \d+ \d+:\d+:\d+\.\d{6}) (\w+) (\w+):/m,
|
34
|
+
:timestamp, :host, :pid).
|
35
|
+
|
36
|
+
# Group messages from a single request together.
|
37
|
+
# Since the logs are from multiple web server processes, messages
|
38
|
+
# might be interleaved with those from other processes. Use the
|
39
|
+
# :pid and :host fields to separate them. Whenever a processes
|
40
|
+
# logs "Processing," that's a new request.
|
41
|
+
group(:following => [:pid, :host], :from => /: Processing/).
|
42
|
+
|
43
|
+
# Pull some per-request fields out of the requests.
|
44
|
+
fields(/Processing (\w+)#(\w+)/, :controller, :action).
|
45
|
+
fields(/Session ID: (\w+)/, :session_id)
|
46
|
+
|
47
|
+
# Filter by a field. Note this returns a new filter immediately
|
48
|
+
# but does not consume any input yet.
|
49
|
+
accounts_requests = requests.where(:controller => "AccountsController")
|
50
|
+
|
51
|
+
# Consume the entire stream and count the number of events
|
52
|
+
accounts_requests.count
|
53
|
+
|
54
|
+
# View the first five matches with "less"
|
55
|
+
accounts_requests.head(5).less
|
56
|
+
|
57
|
+
# Group requests into session traces
|
58
|
+
sessions = requests.group(:following => :session_id)
|
59
|
+
|
60
|
+
# Turn the timestamp into a real ruby object
|
61
|
+
ts_requests = requests.parse_time(:timestamp)
|
62
|
+
|
63
|
+
# Shuffle two streams into one, with events in order of timestamp
|
64
|
+
ts_requests.interleave(other_host_requests)
|
65
|
+
</code>
|
66
|
+
</pre>
|
67
|
+
|
68
|
+
Wolverine works well interactively:
|
69
|
+
|
70
|
+
bc. $ irb -r wolverine
|
71
|
+
|
72
|
+
You may also wish to store some helper functions specific to your log format and/or task at hand to pre-load into your session:
|
73
|
+
|
74
|
+
bc. $ irb -r my-app
|
75
|
+
|
76
|
+
|
77
|
+
h2. Road Map
|
78
|
+
|
79
|
+
* Sources to read files over SSH
|
80
|
+
* Sources that can tail a file
|
81
|
+
* Histograms and tables
|
82
|
+
* Faster field extraction (regex matching is the bottleneck)
|
83
|
+
* Processing multiple pipelines in one pass over the underlying data
|
84
|
+
* Sliding window filters that emit when a 5-minute average exceeds a threshold
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'wolverine'
|
data/lib/wolverine.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module Wolverine
|
2
|
+
end
|
3
|
+
|
4
|
+
require 'wolverine/filter_dsl'
|
5
|
+
require 'wolverine/enumerable'
|
6
|
+
require 'wolverine/source'
|
7
|
+
require 'wolverine/file_source'
|
8
|
+
require 'wolverine/active_record_source'
|
9
|
+
require 'wolverine/array_source'
|
10
|
+
require 'wolverine/filter'
|
11
|
+
require 'wolverine/append_indented_filter'
|
12
|
+
require 'wolverine/field_filter'
|
13
|
+
require 'wolverine/parse_time_filter'
|
14
|
+
require 'wolverine/head_filter'
|
15
|
+
require 'wolverine/where_filter'
|
16
|
+
require 'wolverine/group_filter'
|
17
|
+
require 'wolverine/interleave_filter'
|
18
|
+
require 'wolverine/sink'
|
19
|
+
require 'wolverine/count_sink'
|
20
|
+
require 'wolverine/less_sink'
|
21
|
+
require 'wolverine/file_sink'
|
22
|
+
require 'wolverine/active_record_sink'
|
23
|
+
require 'wolverine/event'
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class ActiveRecordSink < Sink
|
3
|
+
attr_reader :columns, :klass
|
4
|
+
def initialize(source, klass)
|
5
|
+
super(source)
|
6
|
+
@klass = klass
|
7
|
+
@columns = klass.columns.map(&:name) - %w{id created_at updated_at}
|
8
|
+
end
|
9
|
+
def self.create(source, table_name, columns, connection_params)
|
10
|
+
require 'active_record'
|
11
|
+
klass = Class.new(ActiveRecord::Base)
|
12
|
+
klass.class_eval { self.table_name = table_name }
|
13
|
+
klass.establish_connection(connection_params)
|
14
|
+
klass.connection.create_table table_name do |t|
|
15
|
+
t.column :text, :text
|
16
|
+
columns.each do |col|
|
17
|
+
t.column col, :string
|
18
|
+
end
|
19
|
+
#t.timestamps
|
20
|
+
end
|
21
|
+
self.new(source, klass)
|
22
|
+
end
|
23
|
+
def run
|
24
|
+
@source.each do |evt|
|
25
|
+
hsh = {}
|
26
|
+
columns.each {|col| hsh[col] = evt.send(col) }
|
27
|
+
rec = @klass.new(hsh)
|
28
|
+
rec.save!
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class ActiveRecordSource < Source
|
3
|
+
attr_reader :klass
|
4
|
+
def initialize(active_record_klass, conditions={}, search=nil)
|
5
|
+
@klass = active_record_klass
|
6
|
+
@conds = conditions
|
7
|
+
@search = search
|
8
|
+
require 'logger'
|
9
|
+
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
10
|
+
end
|
11
|
+
def self.open(table_name, connection_params)
|
12
|
+
require 'active_record'
|
13
|
+
klass = Class.new(ActiveRecord::Base)
|
14
|
+
klass.class_eval do
|
15
|
+
self.table_name = table_name
|
16
|
+
def to_s
|
17
|
+
text
|
18
|
+
end
|
19
|
+
end
|
20
|
+
klass.establish_connection(connection_params)
|
21
|
+
self.new(klass)
|
22
|
+
end
|
23
|
+
def each
|
24
|
+
if can_search_with_conditions
|
25
|
+
evts = @klass.where({})
|
26
|
+
evts = evts.where(@conds) if !@conds.empty?
|
27
|
+
if @search
|
28
|
+
evts = evts.where(["text like ?", "%#{@search}%"])
|
29
|
+
# Add a fulltext search if we're on MySQL. Boolean fulltext search
|
30
|
+
# is slightly less strict than "like", so it will only be a prefilter.
|
31
|
+
# Fixme: table could be Innodb...
|
32
|
+
if !@search.include? '"' and
|
33
|
+
evts.connection.class.name.include? "Mysql"
|
34
|
+
evts = evts.where(["MATCH (text) AGAINST (? IN BOOLEAN MODE)",
|
35
|
+
"\"#{@search}\""])
|
36
|
+
end
|
37
|
+
end
|
38
|
+
else
|
39
|
+
if @search
|
40
|
+
evts = @klass.find(:all, :conditions =>
|
41
|
+
["text like ?", "%#{@search}%"])
|
42
|
+
else
|
43
|
+
evts = @klass.find(:all, :conditions => @conds)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
evts.each {|rec| yield rec }
|
47
|
+
end
|
48
|
+
def where(conditions=nil)
|
49
|
+
if @search && !can_search_with_conditions
|
50
|
+
raise Exception, "Can't do both where and search"
|
51
|
+
end
|
52
|
+
|
53
|
+
if conditions
|
54
|
+
conds = @conds.merge(conditions)
|
55
|
+
return self.class.new(@klass, conds, @search)
|
56
|
+
else
|
57
|
+
# Get ready for where.not(...), which should happen in-memory
|
58
|
+
super(conditions)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
def search(word)
|
62
|
+
if @conds && !can_search_with_conditions
|
63
|
+
raise Exception, "Can't do both search and where"
|
64
|
+
end
|
65
|
+
self.class.new(@klass, @conds, word)
|
66
|
+
end
|
67
|
+
def can_search_with_conditions
|
68
|
+
@klass.respond_to? :where
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class AppendIndentedFilter < Filter
|
3
|
+
def each
|
4
|
+
merged_evt = nil
|
5
|
+
prefix = self.prefix
|
6
|
+
@source.each do |evt|
|
7
|
+
if evt.to_s.start_with? prefix
|
8
|
+
if merged_evt
|
9
|
+
merged_evt = merge_events(merged_evt, evt)
|
10
|
+
# else drop the partial event
|
11
|
+
end
|
12
|
+
else
|
13
|
+
if merged_evt
|
14
|
+
yield merged_evt
|
15
|
+
end
|
16
|
+
merged_evt = evt
|
17
|
+
end
|
18
|
+
end
|
19
|
+
if merged_evt
|
20
|
+
yield merged_evt
|
21
|
+
end
|
22
|
+
end
|
23
|
+
def merge_events(ev1, ev2)
|
24
|
+
str = ev2.to_s
|
25
|
+
return ev1.to_s + str.slice(prefix.length..str.length)
|
26
|
+
end
|
27
|
+
def prefix
|
28
|
+
" "
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class Enumerable
|
3
|
+
include ::Enumerable
|
4
|
+
include FilterDSL
|
5
|
+
def to_strings
|
6
|
+
map {|el| el.to_s}
|
7
|
+
end
|
8
|
+
def to_a
|
9
|
+
filterize super
|
10
|
+
end
|
11
|
+
def map(*args)
|
12
|
+
filterize super
|
13
|
+
end
|
14
|
+
alias collect map
|
15
|
+
|
16
|
+
protected
|
17
|
+
|
18
|
+
def filterize(arr)
|
19
|
+
arr.extend(FilterDSL)
|
20
|
+
def arr.inspect
|
21
|
+
super if size < 1
|
22
|
+
"[#{first.inspect}, (size: #{size})...]"
|
23
|
+
end
|
24
|
+
arr
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class FieldFilter < Filter
|
3
|
+
def initialize(source, regex, *fields)
|
4
|
+
super(source)
|
5
|
+
@regex = regex
|
6
|
+
@fields = fields
|
7
|
+
@class = Class.new(Event)
|
8
|
+
@class.class_eval <<-EOF, __FILE__, __LINE__
|
9
|
+
def initialize(text, evt, #{@fields.join(", ")})
|
10
|
+
super(text)
|
11
|
+
@evt = evt
|
12
|
+
#{@fields.map {|f| "@#{f}" }.join(", ")} = #{@fields.join(", ")}
|
13
|
+
end
|
14
|
+
def method_missing(name, *args)
|
15
|
+
# Danger: bypassing method access modifier (private/protected)
|
16
|
+
@evt.send(name) if args.empty?
|
17
|
+
end
|
18
|
+
EOF
|
19
|
+
@class.send(:attr_reader, *@fields)
|
20
|
+
# Explicitly enumerating the variables passed is faster than splat
|
21
|
+
self.instance_eval <<-EOF, __FILE__, __LINE__
|
22
|
+
def self.each
|
23
|
+
@source.each do |evt|
|
24
|
+
md = @regex.match(evt.to_s) || []
|
25
|
+
#md = ["pepper.bp", "12345"]
|
26
|
+
#{@fields.join(", ")} =
|
27
|
+
#{@fields.length > 1 ? "md[1..#{@fields.length}]" : "md[1]"}
|
28
|
+
yield @class.new(evt.to_s, evt, #{@fields.join(", ")})
|
29
|
+
end
|
30
|
+
end
|
31
|
+
EOF
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'progressbar'
|
2
|
+
module Wolverine
|
3
|
+
class FileSource < Source
|
4
|
+
attr_accessor :progress, :gzip
|
5
|
+
def initialize(filename, opts={})
|
6
|
+
@filename = filename
|
7
|
+
self.progress = opts[:progress]
|
8
|
+
self.gzip = opts.key?(:gzip) ? opts[:gzip] : filename.match(/\.gz$/i)
|
9
|
+
require 'zlib' if self.gzip
|
10
|
+
end
|
11
|
+
def each
|
12
|
+
progress = self.progress
|
13
|
+
if progress
|
14
|
+
size = File.size(@filename)
|
15
|
+
bar = ProgressBar.new(@filename, size)
|
16
|
+
bar.file_transfer_mode
|
17
|
+
end
|
18
|
+
File.open(@filename, "r") do |file|
|
19
|
+
realfile = file
|
20
|
+
file = Zlib::GzipReader.new(file) if self.gzip
|
21
|
+
bar_update = 10_000
|
22
|
+
cnt = 0
|
23
|
+
file.each do |line|
|
24
|
+
if progress
|
25
|
+
if cnt < bar_update
|
26
|
+
cnt += 1
|
27
|
+
else
|
28
|
+
bar.set(realfile.tell)
|
29
|
+
cnt = 0
|
30
|
+
end
|
31
|
+
end
|
32
|
+
yield line
|
33
|
+
end
|
34
|
+
end
|
35
|
+
bar.finish if progress
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Wolverine
|
2
|
+
module FilterDSL
|
3
|
+
def append_indented
|
4
|
+
AppendIndentedFilter.new(self)
|
5
|
+
end
|
6
|
+
def field(regex, *fields)
|
7
|
+
FieldFilter.new(self, regex, *fields)
|
8
|
+
end
|
9
|
+
def parse_time(source_field, dest_field=source_field)
|
10
|
+
ParseTimeFilter.new(self, source_field, dest_field)
|
11
|
+
end
|
12
|
+
def head(count, opts={})
|
13
|
+
HeadFilter.new(self, count, opts)
|
14
|
+
end
|
15
|
+
def where(conditions=nil)
|
16
|
+
WhereFilter.where(self, conditions)
|
17
|
+
end
|
18
|
+
def count
|
19
|
+
CountSink.new(self).run
|
20
|
+
end
|
21
|
+
def less
|
22
|
+
LessSink.new(self).run
|
23
|
+
end
|
24
|
+
def save_file(filename)
|
25
|
+
FileSink.new(self, filename).run
|
26
|
+
end
|
27
|
+
def save_records(active_record_klass)
|
28
|
+
ActiveRecordSink.new(self, active_record_klass).run
|
29
|
+
end
|
30
|
+
def group(opts)
|
31
|
+
GroupFilter.new(self, opts)
|
32
|
+
end
|
33
|
+
def interleave(other_source)
|
34
|
+
InterleaveFilter.new(self, other_source)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class GroupFilter < Filter
|
3
|
+
def initialize(source, opts)
|
4
|
+
super(source)
|
5
|
+
@following = arg_to_a(opts[:following])
|
6
|
+
@from = opts[:from]
|
7
|
+
end
|
8
|
+
def each
|
9
|
+
groups = Hash.new { |hash,k| hash[k] = [] }
|
10
|
+
@source.each do |evt|
|
11
|
+
key = @following.map {|meth| evt.send(meth) }
|
12
|
+
group = groups[key]
|
13
|
+
if @from
|
14
|
+
if @from.match(evt.to_s)
|
15
|
+
if group.any?
|
16
|
+
yield merge_events(group)
|
17
|
+
group.clear
|
18
|
+
end
|
19
|
+
group.push(evt)
|
20
|
+
else
|
21
|
+
group.push(evt) unless group.empty?
|
22
|
+
end
|
23
|
+
else
|
24
|
+
group.push(evt)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
# ordering?
|
28
|
+
groups.each do |k,group|
|
29
|
+
yield merge_events(group)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
private
|
33
|
+
def arg_to_a(arg)
|
34
|
+
if Array === arg
|
35
|
+
arg
|
36
|
+
else
|
37
|
+
[arg]
|
38
|
+
end
|
39
|
+
end
|
40
|
+
def merge_events(group)
|
41
|
+
Event.new(group.map {|evt| evt.to_s }.join(""))
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'progressbar'
|
2
|
+
module Wolverine
|
3
|
+
class GzipFileSource < Source
|
4
|
+
attr_accessor :progress
|
5
|
+
def initialize(filename, opts={})
|
6
|
+
@filename = filename
|
7
|
+
self.progress = opts[:progress]
|
8
|
+
end
|
9
|
+
def each
|
10
|
+
progress = self.progress
|
11
|
+
if progress
|
12
|
+
size = File.size(@filename)
|
13
|
+
bar = ProgressBar.new(@filename, size)
|
14
|
+
bar.file_transfer_mode
|
15
|
+
end
|
16
|
+
File.open(@filename, "r") do |file|
|
17
|
+
bar_update = 10_000
|
18
|
+
cnt = 0
|
19
|
+
file.each do |line|
|
20
|
+
if progress
|
21
|
+
if cnt < bar_update
|
22
|
+
cnt += 1
|
23
|
+
else
|
24
|
+
bar.set(file.tell)
|
25
|
+
cnt = 0
|
26
|
+
end
|
27
|
+
end
|
28
|
+
yield Event.new(line)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
bar.finish if progress
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class HeadFilter < Filter
|
3
|
+
attr_accessor :progress
|
4
|
+
def initialize(source, limit, opts={})
|
5
|
+
super(source)
|
6
|
+
@limit = limit
|
7
|
+
self.progress = opts[:progress]
|
8
|
+
end
|
9
|
+
def each
|
10
|
+
progress = self.progress
|
11
|
+
bar = ProgressBar.new("head(#{number_with_delimiter @limit})",
|
12
|
+
@limit) if progress
|
13
|
+
cnt = 0
|
14
|
+
@source.each do |evt|
|
15
|
+
bar.set(cnt) if progress
|
16
|
+
if cnt >= @limit
|
17
|
+
bar.finish if progress
|
18
|
+
return
|
19
|
+
end
|
20
|
+
yield evt
|
21
|
+
cnt += 1
|
22
|
+
end
|
23
|
+
end
|
24
|
+
# From Rails.
|
25
|
+
def number_with_delimiter(number, delimiter=",")
|
26
|
+
number.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'generator'
|
2
|
+
|
3
|
+
module Wolverine
|
4
|
+
class InterleaveFilter < Filter
|
5
|
+
def initialize(source, other_source)
|
6
|
+
super(source)
|
7
|
+
@other_source = other_source
|
8
|
+
end
|
9
|
+
def each(&block)
|
10
|
+
# The Generator docs warn these are slow b/c they use continuations
|
11
|
+
generators = [Generator.new(@source), Generator.new(@other_source)]
|
12
|
+
generators.reject!(&:end?)
|
13
|
+
while generators.any?
|
14
|
+
generators = generators.sort_by(&:current)
|
15
|
+
yield generators.first.next
|
16
|
+
generators.shift if generators.first.end?
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class ParseTimeFilter < FieldFilter
|
3
|
+
def initialize(source, source_field, dest_field=source_field)
|
4
|
+
@source_field = source_field
|
5
|
+
@dest_field = dest_field
|
6
|
+
super(source, nil, dest_field)
|
7
|
+
# each is defined by eval in superclass
|
8
|
+
#self.send(:alias_method, :each, :_each)
|
9
|
+
def self.each(*args, &block)
|
10
|
+
_each(*args, &block)
|
11
|
+
end
|
12
|
+
@class.class_eval <<-EOF, __FILE__, __LINE__
|
13
|
+
def <=>(other)
|
14
|
+
mine = self.#@dest_field
|
15
|
+
yours = other.#@dest_field
|
16
|
+
|
17
|
+
# Handle nils; sort them first
|
18
|
+
if mine.nil?
|
19
|
+
if yours.nil?
|
20
|
+
return 0
|
21
|
+
end
|
22
|
+
return -1
|
23
|
+
end
|
24
|
+
if yours.nil?
|
25
|
+
return 1
|
26
|
+
end
|
27
|
+
|
28
|
+
mine <=> yours
|
29
|
+
end
|
30
|
+
EOF
|
31
|
+
end
|
32
|
+
|
33
|
+
def _each
|
34
|
+
@source.each do |evt|
|
35
|
+
ts_text = evt.send(@source_field)
|
36
|
+
ts = Time.parse(ts_text) if ts_text
|
37
|
+
yield @class.new(evt.to_s, evt, ts)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module Wolverine
|
2
|
+
class WhereFilter < Filter
|
3
|
+
attr_reader :conditions
|
4
|
+
def initialize(source, conditions)
|
5
|
+
super(source)
|
6
|
+
@conditions = Conditions.new(self, conditions)
|
7
|
+
end
|
8
|
+
def each
|
9
|
+
@source.each do |evt|
|
10
|
+
next unless @conditions.match?(evt)
|
11
|
+
yield evt
|
12
|
+
end
|
13
|
+
end
|
14
|
+
def self.where(source, conditions=nil)
|
15
|
+
where = WhereFilter.new(source, conditions)
|
16
|
+
if conditions
|
17
|
+
where
|
18
|
+
else
|
19
|
+
where.conditions
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class Conditions
|
25
|
+
def initialize(where, conds=nil)
|
26
|
+
@where = where
|
27
|
+
set_conds(conds)
|
28
|
+
end
|
29
|
+
def match?(evt)
|
30
|
+
@conditions_hash.each do |k,v|
|
31
|
+
return @negate unless v === evt.send(k)
|
32
|
+
end
|
33
|
+
return !@negate
|
34
|
+
end
|
35
|
+
def not(conds)
|
36
|
+
set_conds(conds)
|
37
|
+
@negate = true
|
38
|
+
@where
|
39
|
+
end
|
40
|
+
private
|
41
|
+
def set_conds(conds)
|
42
|
+
return unless conds
|
43
|
+
@conditions_hash = case conds
|
44
|
+
when Regexp then {:to_s => conds}
|
45
|
+
when Hash then conds
|
46
|
+
else raise ArgumentError, "Unknown parameter type: #{conds.class}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
metadata
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: winsome_wolverine
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 0
|
9
|
+
- 1
|
10
|
+
version: 0.0.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Marcel M. Cary
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-11-28 00:00:00 -08:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description: Library and DSL to process log files with a pipe-and-filter architecture
|
23
|
+
email: marcel@oak.homeunix.org
|
24
|
+
executables: []
|
25
|
+
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files: []
|
29
|
+
|
30
|
+
files:
|
31
|
+
- README.textile
|
32
|
+
- lib/winsome_wolverine.rb
|
33
|
+
- lib/wolverine/active_record_sink.rb
|
34
|
+
- lib/wolverine/active_record_source.rb
|
35
|
+
- lib/wolverine/append_indented_filter.rb
|
36
|
+
- lib/wolverine/array_source.rb
|
37
|
+
- lib/wolverine/core_ext.rb
|
38
|
+
- lib/wolverine/count_sink.rb
|
39
|
+
- lib/wolverine/enumerable.rb
|
40
|
+
- lib/wolverine/event.rb
|
41
|
+
- lib/wolverine/field_filter.rb
|
42
|
+
- lib/wolverine/file_sink.rb
|
43
|
+
- lib/wolverine/file_source.rb
|
44
|
+
- lib/wolverine/Filter.rb
|
45
|
+
- lib/wolverine/filter_dsl.rb
|
46
|
+
- lib/wolverine/group_filter.rb
|
47
|
+
- lib/wolverine/gzip_file_source.rb
|
48
|
+
- lib/wolverine/head_filter.rb
|
49
|
+
- lib/wolverine/interleave_filter.rb
|
50
|
+
- lib/wolverine/less_sink.rb
|
51
|
+
- lib/wolverine/parse_time_filter.rb
|
52
|
+
- lib/wolverine/sink.rb
|
53
|
+
- lib/wolverine/source.rb
|
54
|
+
- lib/wolverine/where_filter.rb
|
55
|
+
- lib/wolverine.rb
|
56
|
+
has_rdoc: true
|
57
|
+
homepage: http://github.com/mcary/wolverine
|
58
|
+
licenses: []
|
59
|
+
|
60
|
+
post_install_message:
|
61
|
+
rdoc_options: []
|
62
|
+
|
63
|
+
require_paths:
|
64
|
+
- lib
|
65
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
hash: 3
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
version: "0"
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
hash: 3
|
80
|
+
segments:
|
81
|
+
- 0
|
82
|
+
version: "0"
|
83
|
+
requirements: []
|
84
|
+
|
85
|
+
rubyforge_project:
|
86
|
+
rubygems_version: 1.4.2
|
87
|
+
signing_key:
|
88
|
+
specification_version: 3
|
89
|
+
summary: Log file processor
|
90
|
+
test_files: []
|
91
|
+
|