evidence 0.0.1 → 0.0.2

data/examples/mingle_logs_analysis.rb

@@ -1,5 +1,6 @@
 require 'evidence'
 require 'time'
+require 'fileutils'
 
 include Evidence
 
@@ -14,53 +15,23 @@ mingle_log_pattern = /^
   \[tenant\:(?<tenant>[^\]]*)\]\s+
   (?<message>.*)
 $/x
-mingle_bg_log_pattern = /^
-  \w{3}\s+\d+\s+\d{2}\:\d{2}\:\d{2}\s+
-  (?<host_name>[^\s]+)\s+
-  [\w-_]+\:\s+
-  INFO\s+
-  \[(?<timestamp>[^\]]+)\]\s+
-  \[[\w-_\d]+\[(?<thread_label>[^\]]+)\]\]\s+
-  \[(?<log4j_label>[^\]]+)\]\s+
-  \[tenant\:(?<tenant>[^\]]*)\]\s+
-  (?<message>.*)
-$/x
 
 pid = lambda {|log| "#{log[:host_name]}-#{log[:thread_label]}"}
 message = lambda {|log| log[:message]}
 
-def parse_timestamp
-  lambda do |block|
-    lambda do |log|
-      block.call(log.merge(timestamp: Time.parse(log[:timestamp])))
-    end
-  end
-end
-
-logs = Dir['./../mingle-saas/log/dumpling/**/mingle-cluster*'].reject do |f|
-  File.new(f).first(100).any? do |l|
-    l =~ mingle_bg_log_pattern
-  end
-end.map do |f|
-  stream(File.new(f), log_parser(mingle_log_pattern))
-end
+logs = Dir['./log/**/*'].map { |f| stream(File.new(f)) | log_parser(mingle_log_pattern) }
+time_window = (ARGV[0] || 60).to_i
 puts "[DEBUG]logs.size => #{logs.size.inspect}"
+puts "[DEBUG] time_window => #{time_window.inspect}"
 
-merged = merge_streams(logs, lambda {|log1, log2| log1[:timestamp] <=> log2[:timestamp]})
+merged_logs = merge_streams(logs, lambda {|log1, log2| log1[:timestamp] <=> log2[:timestamp]})
+result = merged_logs | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, time_window) | littles_law_analysis
 
-#counter = 0
-
-#stream(merged, lambda {|block| lambda {|log| block.call(counter += 1)}}).each do |count|
-#   puts "#{Time.now}: #{count}"
-#end
-
-actions_stream = stream(merged, rails_action_parser(pid, message))
-
-time_window = (ARGV[0] || 60).to_i
-File.open('datafile', 'w') do |f|
-  stream(actions_stream, littles_law_analysis(time_window)).map do |start_time, end_time, avg_stay_in_system|
-    t1 = start_time.strftime("%m-%d %H:%M")
-    t2 = end_time.strftime("%H:%M")
+FileUtils.mkdir_p('out')
+File.open('out/mingle_logs_littles_law_analysis', 'w') do |f|
+  result.map do |range, avg_stay_in_system|
+    t1 = range.min.strftime("%m-%d %H:%M")
+    t2 = range.max.strftime("%H:%M")
     f.write("#{t1}-#{t2} #{avg_stay_in_system}\n".tap{|r| puts r})
   end
 end

data/lib/evidence/log_parser.rb

@@ -4,14 +4,14 @@ module Evidence
       @pattern, @unmatched = pattern, unmatched
     end
 
-    def [](block)
-      single_pattern_parser(block)
+    def [](output)
+      single_pattern_parser(output)
     end
 
-    def single_pattern_parser(block)
+    def single_pattern_parser(output)
       lambda do |line|
         if m = @pattern.match(line)
-          block.call(to_hash(m))
+          output.call(to_hash(m))
         else
           @unmatched.call(line)
         end

data/lib/evidence/rails_action_parser.rb

@@ -6,14 +6,14 @@ module Evidence
       @unmatched = unmatched
     end
 
-    def [](block)
+    def [](output)
       processes = Hash.new
       lambda do |log|
         pid = @pid[log]
         if processes.has_key?(pid)
           processes[pid] << log
           if end_action?(@message[log])
-            block.call(parse_action_logs(processes.delete(pid)))
+            output.call(parse_action_logs(processes.delete(pid)))
           end
         else
           if start_action?(@message[log])

data/lib/evidence/stream.rb

@@ -1,22 +1,32 @@
 module Evidence
+  module Pipeline
+    def |(process)
+      Stream.new(self, process)
+    end
+  end
+
+  # A stream is an Enumerable with a process processing the data comming
+  # from input stream and output as another stream
   class Stream
     include Enumerable
+    include Pipeline
 
-    def initialize(upstream, processor)
-      @upstream, @processor = upstream, processor
+    def initialize(upstream, process)
+      @upstream, @process = upstream, process
     end
 
     def eos?
       @upstream.eos?
     end
 
-    def each(&block)
-      @upstream.each(&@processor[block])
+    def each(&output)
+      @upstream.each(&@process[output])
     end
   end
 
   class FileStream
     include Enumerable
+    include Pipeline
 
     def initialize(file)
       @file = file
@@ -26,13 +36,15 @@ module Evidence
       @file.eof?
     end
 
-    def each(&block)
-      @file.each(&block)
+    def each(&output)
+      @file.each(&output)
     end
   end
 
   class ArrayStream
     include Enumerable
+    include Pipeline
+
     def initialize(array)
       @array = array
     end
@@ -41,9 +53,9 @@ module Evidence
       @array.empty?
     end
 
-    def each(&block)
+    def each(&output)
       while(item = @array.shift) do
-        block.call(item)
+        output.call(item)
       end
     end
 
@@ -54,6 +66,7 @@ module Evidence
 
   class MergedStream
     include Enumerable
+    include Pipeline
 
     def initialize(streams, comparator)
       @comparator = comparator
@@ -64,11 +77,11 @@ module Evidence
       @heads.empty?
     end
 
-    def each(&block)
+    def each(&output)
       pull_heads
       loop do
         if min = @heads.min{|a, b| @comparator.call(a[:element], b[:element])}
-          block.call(min.delete(:element).tap{ pull_heads })
+          output.call(min.delete(:element).tap{ pull_heads })
         else
           break if @heads.empty?
           pull_heads
@@ -92,6 +105,8 @@ module Evidence
 
   class Counter
     include Enumerable
+    include Pipeline
+
     def initialize
       @count = 0
     end
@@ -100,9 +115,9 @@ module Evidence
       false
     end
 
-    def each(&block)
+    def each(&output)
       loop do
-        block.call(@count += 1)
+        output.call(@count += 1)
       end
     end
   end

data/lib/evidence.rb

@@ -1,23 +1,20 @@
 require 'evidence/stream'
 require 'evidence/log_parser'
 require 'evidence/rails_action_parser'
-require 'evidence/littles_law_analysis'
 
 module Evidence
   module_function
 
-  # A stream is an Enumerable with a processor processing the data comming
-  # from upstream and yield to downstream
-  def stream(obj, processor=nil)
-    up_stream = case obj
+  # convert Array or File object to a stream
+  def stream(obj)
+    case obj
     when Array
       ArrayStream.new(obj)
     when File
       FileStream.new(obj)
     else
-      obj
+      raise "Unknown how to convert #{obj.class} to a stream"
     end
-    Stream.new(up_stream, processor || lambda {|b| b})
   end
 
   def merge_streams(streams, comparator)
@@ -29,23 +26,71 @@ module Evidence
     end
   end
 
+  def slice_stream(index, step, start_index=nil)
+    end_index = step.is_a?(Proc) ? step : lambda { |index| index + step }
+    lambda do |output|
+      @cache ||= []
+      start_index ||= @cache.first ? index[@cache.first] : nil
+      lambda do |log|
+        next_index = index[log]
+        start_index = index[log] if start_index.nil?
+        return if start_index > next_index
+        @cache << log
+        if end_index[start_index] <= next_index
+          range = start_index..end_index[start_index]
+          start_index = range.max
+          output.call(range, @cache.shift(@cache.size - 1))
+        end
+      end
+    end
+  end
+
   def counter
     Counter.new
   end
 
-  def log_parser(pattern, unmatched=default_unmatched_processor)
+  # Parse log file stream by given pattern
+  #   pattern: ruby regex expression, has named group specified
+  #   output stream: hash object with name and captured string in log
+  def log_parser(pattern, unmatched=default_unmatched_process)
     LogParser.new(pattern, unmatched)
   end
 
-  def rails_action_parser(pid, message, unmatched=default_unmatched_processor)
+  # Parse out rails actions by given:
+  #   pid: a lambda returns process id used to group logs
+  #   message: a lambda returns rails log string message
+  def rails_action_parser(pid, message, unmatched=default_unmatched_process)
     RailsActionParser.new(pid, message, unmatched)
   end
 
-  def littles_law_analysis(time_window)
-    LittlesLawAnalysis.new(time_window)
+  # Rails action request timestamp parser
+  #   log stream | rails_action_parser(pid, message) | request_timestamp_parser
+  def request_timestamp_parser(format="%Y-%m-%d %H:%M:%S")
+    lambda do |output|
+      lambda do |action|
+        action[:request][:timestamp] = Time.strptime(action[:request][:timestamp], format)
+        output[action]
+      end
+    end
+  end
+
+  # Do the little's law analysis on rails actions stream with request_timestamp_parser
+  # usage example:
+  #   log stream | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, 60) | littles_law_analysis
+  def littles_law_analysis
+    lambda do |output|
+      lambda do |range, logs|
+        count = logs.size
+        avg_response_time = logs.reduce(0) {|memo, log| memo + log[:response][:completed_time].to_i} / count
+
+        avg_sec_arrival_rate = count.to_f/(range.max - range.min)
+        avg_sec_response_time = avg_response_time.to_f/1000
+        output[range, avg_sec_arrival_rate * avg_sec_response_time]
+      end
+    end
   end
 
-  def default_unmatched_processor
+  def default_unmatched_process
     lambda {|log| }
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: evidence
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-10 00:00:00.000000000 Z
+date: 2013-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -37,7 +37,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - README.md
-- lib/evidence/littles_law_analysis.rb
 - lib/evidence/log_parser.rb
 - lib/evidence/rails_action_parser.rb
 - lib/evidence/stream.rb

data/lib/evidence/littles_law_analysis.rb

@@ -1,56 +0,0 @@
-require 'time'
-
-module Evidence
-  # Little's Law analysis, input stream log should include the following data
-  # {:request => {:timestamp}, :response => {:completed_time}}
-  class LittlesLawAnalysis
-    class Result
-      def initialize(start, time_window)
-        @start, @time_window = start, time_window
-        @end = @start + @time_window
-        @arrival_count = 0
-        @response_time = 0
-      end
-
-      def ended?(timestamp)
-        @end <= timestamp
-      end
-
-      def add(millisecond)
-        @arrival_count += 1
-        @response_time += millisecond
-      end
-
-      def next
-        Result.new(@start + @time_window, @time_window)
-      end
-
-      def value
-        avg_sec_arrival_rate = @arrival_count.to_f/@time_window
-        avg_sec_response_time = @response_time.to_f/1000/@arrival_count
-        [@start, @end, avg_sec_arrival_rate * avg_sec_response_time]
-      end
-    end
-
-    # time_window: second
-    def initialize(time_window)
-      @time_window = time_window
-    end
-
-    def [](block)
-      result = nil
-      lambda do |log|
-        timestamp = Time.parse(log[:request][:timestamp])
-        if result.nil?
-          result = Result.new(timestamp, @time_window)
-        else
-          if result.ended?(timestamp)
-            block.call(result.value)
-            result = result.next
-          end
-        end
-        result.add(log[:response][:completed_time].to_i)
-      end
-    end
-  end
-end