evidence 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  require 'evidence'
2
2
  require 'time'
3
+ require 'fileutils'
3
4
 
4
5
  include Evidence
5
6
 
@@ -14,53 +15,23 @@ mingle_log_pattern = /^
14
15
  \[tenant\:(?<tenant>[^\]]*)\]\s+
15
16
  (?<message>.*)
16
17
  $/x
17
- mingle_bg_log_pattern = /^
18
- \w{3}\s+\d+\s+\d{2}\:\d{2}\:\d{2}\s+
19
- (?<host_name>[^\s]+)\s+
20
- [\w-_]+\:\s+
21
- INFO\s+
22
- \[(?<timestamp>[^\]]+)\]\s+
23
- \[[\w-_\d]+\[(?<thread_label>[^\]]+)\]\]\s+
24
- \[(?<log4j_label>[^\]]+)\]\s+
25
- \[tenant\:(?<tenant>[^\]]*)\]\s+
26
- (?<message>.*)
27
- $/x
28
18
 
29
19
  pid = lambda {|log| "#{log[:host_name]}-#{log[:thread_label]}"}
30
20
  message = lambda {|log| log[:message]}
31
21
 
32
- def parse_timestamp
33
- lambda do |block|
34
- lambda do |log|
35
- block.call(log.merge(timestamp: Time.parse(log[:timestamp])))
36
- end
37
- end
38
- end
39
-
40
- logs = Dir['./../mingle-saas/log/dumpling/**/mingle-cluster*'].reject do |f|
41
- File.new(f).first(100).any? do |l|
42
- l =~ mingle_bg_log_pattern
43
- end
44
- end.map do |f|
45
- stream(File.new(f), log_parser(mingle_log_pattern))
46
- end
22
+ logs = Dir['./log/**/*'].map { |f| stream(File.new(f)) | log_parser(mingle_log_pattern) }
23
+ time_window = (ARGV[0] || 60).to_i
47
24
  puts "[DEBUG]logs.size => #{logs.size.inspect}"
25
+ puts "[DEBUG] time_window => #{time_window.inspect}"
48
26
 
49
- merged = merge_streams(logs, lambda {|log1, log2| log1[:timestamp] <=> log2[:timestamp]})
27
+ merged_logs = merge_streams(logs, lambda {|log1, log2| log1[:timestamp] <=> log2[:timestamp]})
28
+ result = merged_logs | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, time_window) | littles_law_analysis
50
29
 
51
- #counter = 0
52
-
53
- #stream(merged, lambda {|block| lambda {|log| block.call(counter += 1)}}).each do |count|
54
- # puts "#{Time.now}: #{count}"
55
- #end
56
-
57
- actions_stream = stream(merged, rails_action_parser(pid, message))
58
-
59
- time_window = (ARGV[0] || 60).to_i
60
- File.open('datafile', 'w') do |f|
61
- stream(actions_stream, littles_law_analysis(time_window)).map do |start_time, end_time, avg_stay_in_system|
62
- t1 = start_time.strftime("%m-%d %H:%M")
63
- t2 = end_time.strftime("%H:%M")
30
+ FileUtils.mkdir_p('out')
31
+ File.open('out/mingle_logs_littles_law_analysis', 'w') do |f|
32
+ result.map do |range, avg_stay_in_system|
33
+ t1 = range.min.strftime("%m-%d %H:%M")
34
+ t2 = range.max.strftime("%H:%M")
64
35
  f.write("#{t1}-#{t2} #{avg_stay_in_system}\n".tap{|r| puts r})
65
36
  end
66
37
  end
@@ -4,14 +4,14 @@ module Evidence
4
4
  @pattern, @unmatched = pattern, unmatched
5
5
  end
6
6
 
7
- def [](block)
8
- single_pattern_parser(block)
7
+ def [](output)
8
+ single_pattern_parser(output)
9
9
  end
10
10
 
11
- def single_pattern_parser(block)
11
+ def single_pattern_parser(output)
12
12
  lambda do |line|
13
13
  if m = @pattern.match(line)
14
- block.call(to_hash(m))
14
+ output.call(to_hash(m))
15
15
  else
16
16
  @unmatched.call(line)
17
17
  end
@@ -6,14 +6,14 @@ module Evidence
6
6
  @unmatched = unmatched
7
7
  end
8
8
 
9
- def [](block)
9
+ def [](output)
10
10
  processes = Hash.new
11
11
  lambda do |log|
12
12
  pid = @pid[log]
13
13
  if processes.has_key?(pid)
14
14
  processes[pid] << log
15
15
  if end_action?(@message[log])
16
- block.call(parse_action_logs(processes.delete(pid)))
16
+ output.call(parse_action_logs(processes.delete(pid)))
17
17
  end
18
18
  else
19
19
  if start_action?(@message[log])
@@ -1,22 +1,32 @@
1
1
  module Evidence
2
+ module Pipeline
3
+ def |(process)
4
+ Stream.new(self, process)
5
+ end
6
+ end
7
+
8
+ # A stream is an Enumerable with a process processing the data comming
9
+ # from input stream and output as another stream
2
10
  class Stream
3
11
  include Enumerable
12
+ include Pipeline
4
13
 
5
- def initialize(upstream, processor)
6
- @upstream, @processor = upstream, processor
14
+ def initialize(upstream, process)
15
+ @upstream, @process = upstream, process
7
16
  end
8
17
 
9
18
  def eos?
10
19
  @upstream.eos?
11
20
  end
12
21
 
13
- def each(&block)
14
- @upstream.each(&@processor[block])
22
+ def each(&output)
23
+ @upstream.each(&@process[output])
15
24
  end
16
25
  end
17
26
 
18
27
  class FileStream
19
28
  include Enumerable
29
+ include Pipeline
20
30
 
21
31
  def initialize(file)
22
32
  @file = file
@@ -26,13 +36,15 @@ module Evidence
26
36
  @file.eof?
27
37
  end
28
38
 
29
- def each(&block)
30
- @file.each(&block)
39
+ def each(&output)
40
+ @file.each(&output)
31
41
  end
32
42
  end
33
43
 
34
44
  class ArrayStream
35
45
  include Enumerable
46
+ include Pipeline
47
+
36
48
  def initialize(array)
37
49
  @array = array
38
50
  end
@@ -41,9 +53,9 @@ module Evidence
41
53
  @array.empty?
42
54
  end
43
55
 
44
- def each(&block)
56
+ def each(&output)
45
57
  while(item = @array.shift) do
46
- block.call(item)
58
+ output.call(item)
47
59
  end
48
60
  end
49
61
 
@@ -54,6 +66,7 @@ module Evidence
54
66
 
55
67
  class MergedStream
56
68
  include Enumerable
69
+ include Pipeline
57
70
 
58
71
  def initialize(streams, comparator)
59
72
  @comparator = comparator
@@ -64,11 +77,11 @@ module Evidence
64
77
  @heads.empty?
65
78
  end
66
79
 
67
- def each(&block)
80
+ def each(&output)
68
81
  pull_heads
69
82
  loop do
70
83
  if min = @heads.min{|a, b| @comparator.call(a[:element], b[:element])}
71
- block.call(min.delete(:element).tap{ pull_heads })
84
+ output.call(min.delete(:element).tap{ pull_heads })
72
85
  else
73
86
  break if @heads.empty?
74
87
  pull_heads
@@ -92,6 +105,8 @@ module Evidence
92
105
 
93
106
  class Counter
94
107
  include Enumerable
108
+ include Pipeline
109
+
95
110
  def initialize
96
111
  @count = 0
97
112
  end
@@ -100,9 +115,9 @@ module Evidence
100
115
  false
101
116
  end
102
117
 
103
- def each(&block)
118
+ def each(&output)
104
119
  loop do
105
- block.call(@count += 1)
120
+ output.call(@count += 1)
106
121
  end
107
122
  end
108
123
  end
data/lib/evidence.rb CHANGED
@@ -1,23 +1,20 @@
1
1
  require 'evidence/stream'
2
2
  require 'evidence/log_parser'
3
3
  require 'evidence/rails_action_parser'
4
- require 'evidence/littles_law_analysis'
5
4
 
6
5
  module Evidence
7
6
  module_function
8
7
 
9
- # A stream is an Enumerable with a processor processing the data comming
10
- # from upstream and yield to downstream
11
- def stream(obj, processor=nil)
12
- up_stream = case obj
8
+ # convert Array or File object to a stream
9
+ def stream(obj)
10
+ case obj
13
11
  when Array
14
12
  ArrayStream.new(obj)
15
13
  when File
16
14
  FileStream.new(obj)
17
15
  else
18
- obj
16
+ raise "Unknown how to convert #{obj.class} to a stream"
19
17
  end
20
- Stream.new(up_stream, processor || lambda {|b| b})
21
18
  end
22
19
 
23
20
  def merge_streams(streams, comparator)
@@ -29,23 +26,71 @@ module Evidence
29
26
  end
30
27
  end
31
28
 
29
+ def slice_stream(index, step, start_index=nil)
30
+ end_index = step.is_a?(Proc) ? step : lambda { |index| index + step }
31
+ lambda do |output|
32
+ @cache ||= []
33
+ start_index ||= @cache.first ? index[@cache.first] : nil
34
+ lambda do |log|
35
+ next_index = index[log]
36
+ start_index = index[log] if start_index.nil?
37
+ return if start_index > next_index
38
+ @cache << log
39
+ if end_index[start_index] <= next_index
40
+ range = start_index..end_index[start_index]
41
+ start_index = range.max
42
+ output.call(range, @cache.shift(@cache.size - 1))
43
+ end
44
+ end
45
+ end
46
+ end
47
+
32
48
  def counter
33
49
  Counter.new
34
50
  end
35
51
 
36
- def log_parser(pattern, unmatched=default_unmatched_processor)
52
+ # Parse log file stream by given pattern
53
+ # pattern: ruby regex expression, has named group specified
54
+ # output stream: hash object with name and captured string in log
55
+ def log_parser(pattern, unmatched=default_unmatched_process)
37
56
  LogParser.new(pattern, unmatched)
38
57
  end
39
58
 
40
- def rails_action_parser(pid, message, unmatched=default_unmatched_processor)
59
+ # Parse out rails actions by given:
60
+ # pid: a lambda returns process id used to group logs
61
+ # message: a lambda returns rails log string message
62
+ def rails_action_parser(pid, message, unmatched=default_unmatched_process)
41
63
  RailsActionParser.new(pid, message, unmatched)
42
64
  end
43
65
 
44
- def littles_law_analysis(time_window)
45
- LittlesLawAnalysis.new(time_window)
66
+ # Rails action request timestamp parser
67
+ # log stream | rails_action_parser(pid, message) | request_timestamp_parser
68
+ def request_timestamp_parser(format="%Y-%m-%d %H:%M:%S")
69
+ lambda do |output|
70
+ lambda do |action|
71
+ action[:request][:timestamp] = Time.strptime(action[:request][:timestamp], format)
72
+ output[action]
73
+ end
74
+ end
75
+ end
76
+
77
+ # Do the little's law analysis on rails actions stream with request_timestamp_parser
78
+ # usage example:
79
+ # log stream | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, 60) | littles_law_analysis
80
+ def littles_law_analysis
81
+ lambda do |output|
82
+ lambda do |range, logs|
83
+ count = logs.size
84
+ avg_response_time = logs.reduce(0) {|memo, log| memo + log[:response][:completed_time].to_i} / count
85
+
86
+ avg_sec_arrival_rate = count.to_f/(range.max - range.min)
87
+ avg_sec_response_time = avg_response_time.to_f/1000
88
+ output[range, avg_sec_arrival_rate * avg_sec_response_time]
89
+ end
90
+ end
46
91
  end
47
92
 
48
- def default_unmatched_processor
93
+ def default_unmatched_process
49
94
  lambda {|log| }
50
95
  end
51
96
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: evidence
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2013-08-10 00:00:00.000000000 Z
13
+ date: 2013-08-11 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: rake
@@ -37,7 +37,6 @@ extensions: []
37
37
  extra_rdoc_files: []
38
38
  files:
39
39
  - README.md
40
- - lib/evidence/littles_law_analysis.rb
41
40
  - lib/evidence/log_parser.rb
42
41
  - lib/evidence/rails_action_parser.rb
43
42
  - lib/evidence/stream.rb
@@ -1,56 +0,0 @@
1
- require 'time'
2
-
3
- module Evidence
4
- # Little's Law analysis, input stream log should include the following data
5
- # {:request => {:timestamp}, :response => {:completed_time}}
6
- class LittlesLawAnalysis
7
- class Result
8
- def initialize(start, time_window)
9
- @start, @time_window = start, time_window
10
- @end = @start + @time_window
11
- @arrival_count = 0
12
- @response_time = 0
13
- end
14
-
15
- def ended?(timestamp)
16
- @end <= timestamp
17
- end
18
-
19
- def add(millisecond)
20
- @arrival_count += 1
21
- @response_time += millisecond
22
- end
23
-
24
- def next
25
- Result.new(@start + @time_window, @time_window)
26
- end
27
-
28
- def value
29
- avg_sec_arrival_rate = @arrival_count.to_f/@time_window
30
- avg_sec_response_time = @response_time.to_f/1000/@arrival_count
31
- [@start, @end, avg_sec_arrival_rate * avg_sec_response_time]
32
- end
33
- end
34
-
35
- # time_window: second
36
- def initialize(time_window)
37
- @time_window = time_window
38
- end
39
-
40
- def [](block)
41
- result = nil
42
- lambda do |log|
43
- timestamp = Time.parse(log[:request][:timestamp])
44
- if result.nil?
45
- result = Result.new(timestamp, @time_window)
46
- else
47
- if result.ended?(timestamp)
48
- block.call(result.value)
49
- result = result.next
50
- end
51
- end
52
- result.add(log[:response][:completed_time].to_i)
53
- end
54
- end
55
- end
56
- end