evidence 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/examples/mingle_logs_analysis.rb +11 -40
- data/lib/evidence/log_parser.rb +4 -4
- data/lib/evidence/rails_action_parser.rb +2 -2
- data/lib/evidence/stream.rb +27 -12
- data/lib/evidence.rb +57 -12
- metadata +2 -3
- data/lib/evidence/littles_law_analysis.rb +0 -56
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'evidence'
|
2
2
|
require 'time'
|
3
|
+
require 'fileutils'
|
3
4
|
|
4
5
|
include Evidence
|
5
6
|
|
@@ -14,53 +15,23 @@ mingle_log_pattern = /^
|
|
14
15
|
\[tenant\:(?<tenant>[^\]]*)\]\s+
|
15
16
|
(?<message>.*)
|
16
17
|
$/x
|
17
|
-
mingle_bg_log_pattern = /^
|
18
|
-
\w{3}\s+\d+\s+\d{2}\:\d{2}\:\d{2}\s+
|
19
|
-
(?<host_name>[^\s]+)\s+
|
20
|
-
[\w-_]+\:\s+
|
21
|
-
INFO\s+
|
22
|
-
\[(?<timestamp>[^\]]+)\]\s+
|
23
|
-
\[[\w-_\d]+\[(?<thread_label>[^\]]+)\]\]\s+
|
24
|
-
\[(?<log4j_label>[^\]]+)\]\s+
|
25
|
-
\[tenant\:(?<tenant>[^\]]*)\]\s+
|
26
|
-
(?<message>.*)
|
27
|
-
$/x
|
28
18
|
|
29
19
|
pid = lambda {|log| "#{log[:host_name]}-#{log[:thread_label]}"}
|
30
20
|
message = lambda {|log| log[:message]}
|
31
21
|
|
32
|
-
|
33
|
-
|
34
|
-
lambda do |log|
|
35
|
-
block.call(log.merge(timestamp: Time.parse(log[:timestamp])))
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
logs = Dir['./../mingle-saas/log/dumpling/**/mingle-cluster*'].reject do |f|
|
41
|
-
File.new(f).first(100).any? do |l|
|
42
|
-
l =~ mingle_bg_log_pattern
|
43
|
-
end
|
44
|
-
end.map do |f|
|
45
|
-
stream(File.new(f), log_parser(mingle_log_pattern))
|
46
|
-
end
|
22
|
+
logs = Dir['./log/**/*'].map { |f| stream(File.new(f)) | log_parser(mingle_log_pattern) }
|
23
|
+
time_window = (ARGV[0] || 60).to_i
|
47
24
|
puts "[DEBUG]logs.size => #{logs.size.inspect}"
|
25
|
+
puts "[DEBUG] time_window => #{time_window.inspect}"
|
48
26
|
|
49
|
-
|
27
|
+
merged_logs = merge_streams(logs, lambda {|log1, log2| log1[:timestamp] <=> log2[:timestamp]})
|
28
|
+
result = merged_logs | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, time_window) | littles_law_analysis
|
50
29
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
actions_stream = stream(merged, rails_action_parser(pid, message))
|
58
|
-
|
59
|
-
time_window = (ARGV[0] || 60).to_i
|
60
|
-
File.open('datafile', 'w') do |f|
|
61
|
-
stream(actions_stream, littles_law_analysis(time_window)).map do |start_time, end_time, avg_stay_in_system|
|
62
|
-
t1 = start_time.strftime("%m-%d %H:%M")
|
63
|
-
t2 = end_time.strftime("%H:%M")
|
30
|
+
FileUtils.mkdir_p('out')
|
31
|
+
File.open('out/mingle_logs_littles_law_analysis', 'w') do |f|
|
32
|
+
result.map do |range, avg_stay_in_system|
|
33
|
+
t1 = range.min.strftime("%m-%d %H:%M")
|
34
|
+
t2 = range.max.strftime("%H:%M")
|
64
35
|
f.write("#{t1}-#{t2} #{avg_stay_in_system}\n".tap{|r| puts r})
|
65
36
|
end
|
66
37
|
end
|
data/lib/evidence/log_parser.rb
CHANGED
@@ -4,14 +4,14 @@ module Evidence
|
|
4
4
|
@pattern, @unmatched = pattern, unmatched
|
5
5
|
end
|
6
6
|
|
7
|
-
def [](
|
8
|
-
single_pattern_parser(
|
7
|
+
def [](output)
|
8
|
+
single_pattern_parser(output)
|
9
9
|
end
|
10
10
|
|
11
|
-
def single_pattern_parser(
|
11
|
+
def single_pattern_parser(output)
|
12
12
|
lambda do |line|
|
13
13
|
if m = @pattern.match(line)
|
14
|
-
|
14
|
+
output.call(to_hash(m))
|
15
15
|
else
|
16
16
|
@unmatched.call(line)
|
17
17
|
end
|
@@ -6,14 +6,14 @@ module Evidence
|
|
6
6
|
@unmatched = unmatched
|
7
7
|
end
|
8
8
|
|
9
|
-
def [](
|
9
|
+
def [](output)
|
10
10
|
processes = Hash.new
|
11
11
|
lambda do |log|
|
12
12
|
pid = @pid[log]
|
13
13
|
if processes.has_key?(pid)
|
14
14
|
processes[pid] << log
|
15
15
|
if end_action?(@message[log])
|
16
|
-
|
16
|
+
output.call(parse_action_logs(processes.delete(pid)))
|
17
17
|
end
|
18
18
|
else
|
19
19
|
if start_action?(@message[log])
|
data/lib/evidence/stream.rb
CHANGED
@@ -1,22 +1,32 @@
|
|
1
1
|
module Evidence
|
2
|
+
module Pipeline
|
3
|
+
def |(process)
|
4
|
+
Stream.new(self, process)
|
5
|
+
end
|
6
|
+
end
|
7
|
+
|
8
|
+
# A stream is an Enumerable with a process processing the data comming
|
9
|
+
# from input stream and output as another stream
|
2
10
|
class Stream
|
3
11
|
include Enumerable
|
12
|
+
include Pipeline
|
4
13
|
|
5
|
-
def initialize(upstream,
|
6
|
-
@upstream, @
|
14
|
+
def initialize(upstream, process)
|
15
|
+
@upstream, @process = upstream, process
|
7
16
|
end
|
8
17
|
|
9
18
|
def eos?
|
10
19
|
@upstream.eos?
|
11
20
|
end
|
12
21
|
|
13
|
-
def each(&
|
14
|
-
@upstream.each(&@
|
22
|
+
def each(&output)
|
23
|
+
@upstream.each(&@process[output])
|
15
24
|
end
|
16
25
|
end
|
17
26
|
|
18
27
|
class FileStream
|
19
28
|
include Enumerable
|
29
|
+
include Pipeline
|
20
30
|
|
21
31
|
def initialize(file)
|
22
32
|
@file = file
|
@@ -26,13 +36,15 @@ module Evidence
|
|
26
36
|
@file.eof?
|
27
37
|
end
|
28
38
|
|
29
|
-
def each(&
|
30
|
-
@file.each(&
|
39
|
+
def each(&output)
|
40
|
+
@file.each(&output)
|
31
41
|
end
|
32
42
|
end
|
33
43
|
|
34
44
|
class ArrayStream
|
35
45
|
include Enumerable
|
46
|
+
include Pipeline
|
47
|
+
|
36
48
|
def initialize(array)
|
37
49
|
@array = array
|
38
50
|
end
|
@@ -41,9 +53,9 @@ module Evidence
|
|
41
53
|
@array.empty?
|
42
54
|
end
|
43
55
|
|
44
|
-
def each(&
|
56
|
+
def each(&output)
|
45
57
|
while(item = @array.shift) do
|
46
|
-
|
58
|
+
output.call(item)
|
47
59
|
end
|
48
60
|
end
|
49
61
|
|
@@ -54,6 +66,7 @@ module Evidence
|
|
54
66
|
|
55
67
|
class MergedStream
|
56
68
|
include Enumerable
|
69
|
+
include Pipeline
|
57
70
|
|
58
71
|
def initialize(streams, comparator)
|
59
72
|
@comparator = comparator
|
@@ -64,11 +77,11 @@ module Evidence
|
|
64
77
|
@heads.empty?
|
65
78
|
end
|
66
79
|
|
67
|
-
def each(&
|
80
|
+
def each(&output)
|
68
81
|
pull_heads
|
69
82
|
loop do
|
70
83
|
if min = @heads.min{|a, b| @comparator.call(a[:element], b[:element])}
|
71
|
-
|
84
|
+
output.call(min.delete(:element).tap{ pull_heads })
|
72
85
|
else
|
73
86
|
break if @heads.empty?
|
74
87
|
pull_heads
|
@@ -92,6 +105,8 @@ module Evidence
|
|
92
105
|
|
93
106
|
class Counter
|
94
107
|
include Enumerable
|
108
|
+
include Pipeline
|
109
|
+
|
95
110
|
def initialize
|
96
111
|
@count = 0
|
97
112
|
end
|
@@ -100,9 +115,9 @@ module Evidence
|
|
100
115
|
false
|
101
116
|
end
|
102
117
|
|
103
|
-
def each(&
|
118
|
+
def each(&output)
|
104
119
|
loop do
|
105
|
-
|
120
|
+
output.call(@count += 1)
|
106
121
|
end
|
107
122
|
end
|
108
123
|
end
|
data/lib/evidence.rb
CHANGED
@@ -1,23 +1,20 @@
|
|
1
1
|
require 'evidence/stream'
|
2
2
|
require 'evidence/log_parser'
|
3
3
|
require 'evidence/rails_action_parser'
|
4
|
-
require 'evidence/littles_law_analysis'
|
5
4
|
|
6
5
|
module Evidence
|
7
6
|
module_function
|
8
7
|
|
9
|
-
#
|
10
|
-
|
11
|
-
|
12
|
-
up_stream = case obj
|
8
|
+
# convert Array or File object to a stream
|
9
|
+
def stream(obj)
|
10
|
+
case obj
|
13
11
|
when Array
|
14
12
|
ArrayStream.new(obj)
|
15
13
|
when File
|
16
14
|
FileStream.new(obj)
|
17
15
|
else
|
18
|
-
obj
|
16
|
+
raise "Unknown how to convert #{obj.class} to a stream"
|
19
17
|
end
|
20
|
-
Stream.new(up_stream, processor || lambda {|b| b})
|
21
18
|
end
|
22
19
|
|
23
20
|
def merge_streams(streams, comparator)
|
@@ -29,23 +26,71 @@ module Evidence
|
|
29
26
|
end
|
30
27
|
end
|
31
28
|
|
29
|
+
def slice_stream(index, step, start_index=nil)
|
30
|
+
end_index = step.is_a?(Proc) ? step : lambda { |index| index + step }
|
31
|
+
lambda do |output|
|
32
|
+
@cache ||= []
|
33
|
+
start_index ||= @cache.first ? index[@cache.first] : nil
|
34
|
+
lambda do |log|
|
35
|
+
next_index = index[log]
|
36
|
+
start_index = index[log] if start_index.nil?
|
37
|
+
return if start_index > next_index
|
38
|
+
@cache << log
|
39
|
+
if end_index[start_index] <= next_index
|
40
|
+
range = start_index..end_index[start_index]
|
41
|
+
start_index = range.max
|
42
|
+
output.call(range, @cache.shift(@cache.size - 1))
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
32
48
|
def counter
|
33
49
|
Counter.new
|
34
50
|
end
|
35
51
|
|
36
|
-
|
52
|
+
# Parse log file stream by given pattern
|
53
|
+
# pattern: ruby regex expression, has named group specified
|
54
|
+
# output stream: hash object with name and captured string in log
|
55
|
+
def log_parser(pattern, unmatched=default_unmatched_process)
|
37
56
|
LogParser.new(pattern, unmatched)
|
38
57
|
end
|
39
58
|
|
40
|
-
|
59
|
+
# Parse out rails actions by given:
|
60
|
+
# pid: a lambda returns process id used to group logs
|
61
|
+
# message: a lambda returns rails log string message
|
62
|
+
def rails_action_parser(pid, message, unmatched=default_unmatched_process)
|
41
63
|
RailsActionParser.new(pid, message, unmatched)
|
42
64
|
end
|
43
65
|
|
44
|
-
|
45
|
-
|
66
|
+
# Rails action request timestamp parser
|
67
|
+
# log stream | rails_action_parser(pid, message) | request_timestamp_parser
|
68
|
+
def request_timestamp_parser(format="%Y-%m-%d %H:%M:%S")
|
69
|
+
lambda do |output|
|
70
|
+
lambda do |action|
|
71
|
+
action[:request][:timestamp] = Time.strptime(action[:request][:timestamp], format)
|
72
|
+
output[action]
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# Do the little's law analysis on rails actions stream with request_timestamp_parser
|
78
|
+
# usage example:
|
79
|
+
# log stream | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, 60) | littles_law_analysis
|
80
|
+
def littles_law_analysis
|
81
|
+
lambda do |output|
|
82
|
+
lambda do |range, logs|
|
83
|
+
count = logs.size
|
84
|
+
avg_response_time = logs.reduce(0) {|memo, log| memo + log[:response][:completed_time].to_i} / count
|
85
|
+
|
86
|
+
avg_sec_arrival_rate = count.to_f/(range.max - range.min)
|
87
|
+
avg_sec_response_time = avg_response_time.to_f/1000
|
88
|
+
output[range, avg_sec_arrival_rate * avg_sec_response_time]
|
89
|
+
end
|
90
|
+
end
|
46
91
|
end
|
47
92
|
|
48
|
-
def
|
93
|
+
def default_unmatched_process
|
49
94
|
lambda {|log| }
|
50
95
|
end
|
51
96
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: evidence
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2013-08-
|
13
|
+
date: 2013-08-11 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rake
|
@@ -37,7 +37,6 @@ extensions: []
|
|
37
37
|
extra_rdoc_files: []
|
38
38
|
files:
|
39
39
|
- README.md
|
40
|
-
- lib/evidence/littles_law_analysis.rb
|
41
40
|
- lib/evidence/log_parser.rb
|
42
41
|
- lib/evidence/rails_action_parser.rb
|
43
42
|
- lib/evidence/stream.rb
|
@@ -1,56 +0,0 @@
|
|
1
|
-
require 'time'
|
2
|
-
|
3
|
-
module Evidence
|
4
|
-
# Little's Law analysis, input stream log should include the following data
|
5
|
-
# {:request => {:timestamp}, :response => {:completed_time}}
|
6
|
-
class LittlesLawAnalysis
|
7
|
-
class Result
|
8
|
-
def initialize(start, time_window)
|
9
|
-
@start, @time_window = start, time_window
|
10
|
-
@end = @start + @time_window
|
11
|
-
@arrival_count = 0
|
12
|
-
@response_time = 0
|
13
|
-
end
|
14
|
-
|
15
|
-
def ended?(timestamp)
|
16
|
-
@end <= timestamp
|
17
|
-
end
|
18
|
-
|
19
|
-
def add(millisecond)
|
20
|
-
@arrival_count += 1
|
21
|
-
@response_time += millisecond
|
22
|
-
end
|
23
|
-
|
24
|
-
def next
|
25
|
-
Result.new(@start + @time_window, @time_window)
|
26
|
-
end
|
27
|
-
|
28
|
-
def value
|
29
|
-
avg_sec_arrival_rate = @arrival_count.to_f/@time_window
|
30
|
-
avg_sec_response_time = @response_time.to_f/1000/@arrival_count
|
31
|
-
[@start, @end, avg_sec_arrival_rate * avg_sec_response_time]
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
# time_window: second
|
36
|
-
def initialize(time_window)
|
37
|
-
@time_window = time_window
|
38
|
-
end
|
39
|
-
|
40
|
-
def [](block)
|
41
|
-
result = nil
|
42
|
-
lambda do |log|
|
43
|
-
timestamp = Time.parse(log[:request][:timestamp])
|
44
|
-
if result.nil?
|
45
|
-
result = Result.new(timestamp, @time_window)
|
46
|
-
else
|
47
|
-
if result.ended?(timestamp)
|
48
|
-
block.call(result.value)
|
49
|
-
result = result.next
|
50
|
-
end
|
51
|
-
end
|
52
|
-
result.add(log[:response][:completed_time].to_i)
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|