evidence 0.0.4 → 0.0.5

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NTRlNjExNTY0MjllNzYwNjZkZTgyYTA0MGIwN2I3MzcxZDBhYjA3NA==
-  data.tar.gz: !binary |-
-    YjI4MzM1MjQwMzZlYjhiZThjYjhiNGZjNGMyYTU3ZmQ0MjliZTJiYw==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    M2MyMjY3Y2U3Y2IxODRiM2M2NzczODdjYzAxYThiODUwNjNlNDNhZTEyNzUz
-    MjIyMjg0MGMyMTVmZDA1ZDRlNDQ5MDAyOTczMzFjYjU0OTBlMDBmYWE0NWU1
-    ZmE1YWU4MDllYWZmODAxYTUyMzIxMGIwZWIzNDc5NDkzZTQ5ZGE=
-  data.tar.gz: !binary |-
-    Y2Q1MWQwMmIwZjRlMjU3MzQyNTZjYzNjMGQ3NjAwZTAxZGNhNjYxMWYzNjI3
-    MWUxOTdlYTQxM2FmMjg3NTVjY2Y0NGNjOTVhM2FjMjU5OTQ3MjRjZGU0Yzg4
-    NzgwNjJlNjAzOGNkMmFjMTdhOWIwNmM5Zjk5Y2EzNGJmNzg0MjE=
+SHA1:
+  metadata.gz: f2314c07b4b1873ae89834bd97ac81be2f0c1783
+  data.tar.gz: b297e819e4f1792b05dc3a49dde97066378aed13
+SHA512:
+  metadata.gz: 28d755cd6038e0c97bbee4ea1da7ed25940aaa8162887685a0da565b708c30cedcae3720ef0568d1d38fae1ee9560ce2a310fa01dafa3cc02c085b2f7745802d
+  data.tar.gz: 763e35c45812f677a1dc0448b80eab549396913cad12033dca7d5543ea822e63f0945ec746857460483c1dfbf4ebfe179f77bb3d561a7ebb97a7e34352665ff1

data/examples/mingle_logs_analysis.rb

@@ -7,7 +7,7 @@ include Evidence
 mingle_log_pattern = /^
   \w{3}\s+\d+\s+\d{2}\:\d{2}\:\d{2}\s+
   (?<host_name>[^\s]+)\s+
-  [\w-_]+\:\s+
+  [-_\w]+\:\s+
   INFO\s+
   \[(?<timestamp>[^\]]+)\]\s+
   \[(?<thread_label>[^\]]+)\]\s+
@@ -19,19 +19,16 @@ $/x
 pid = lambda {|log| "#{log[:host_name]}-#{log[:thread_label]}"}
 message = lambda {|log| log[:message]}
 
-logs = Dir['./log/**/*'].map { |f| stream(File.new(f)) | log_parser(mingle_log_pattern) }
+logs = Dir['./log/*'].sort.lazy.map { |f| File.new(f).lazy.map(&parse_log(mingle_log_pattern)).compact }.flat_map {|a| a}
 time_window = (ARGV[0] || 60).to_i
 puts "[DEBUG]logs.size => #{logs.size.inspect}"
 puts "[DEBUG] time_window => #{time_window.inspect}"
 
-merged_logs = merge_streams(logs, lambda {|log1, log2| log1[:timestamp] <=> log2[:timestamp]})
-result = merged_logs | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, time_window) | littles_law_analysis
-
 FileUtils.mkdir_p('out')
 File.open('out/mingle_logs_littles_law_analysis', 'w') do |f|
-  result.map do |range, avg_stay_in_system|
-    t1 = range.min.strftime("%m-%d %H:%M")
-    t2 = range.max.strftime("%H:%M")
-    f.write("#{t1}-#{t2} #{avg_stay_in_system}\n".tap{|r| puts r})
-  end
+  logs.map(&rails_action_parser(pid, message)).compact.map(&request_timestamp_parser).chunk(&by_time_window(time_window)).map(&littles_law_analysis).map do |stat|
+    t1 = stat[:range].min.strftime("%m-%d %H:%M")
+    t2 = stat[:range].max.strftime("%H:%M")
+    f.write("#{t1}-#{t2} #{stat[:value]}\n".tap{|r| puts r})
+  end.force
 end

data/lib/evidence/lazy.rb

@@ -0,0 +1,8 @@
+
+module Evidence
+  module Lazy
+    def compact
+      reject(&:nil?)
+    end
+  end
+end

data/lib/evidence/rails_action_parser.rb

@@ -1,26 +1,24 @@
 module Evidence
   class RailsActionParser
 
-    def initialize(pid, message, unmatched)
+    def initialize(pid, message)
       @pid, @message = pid, message
-      @unmatched = unmatched
+      @processes = Hash.new
     end
 
-    def [](output)
-      processes = Hash.new
+    def to_proc
       lambda do |log|
         pid = @pid[log]
-        if processes.has_key?(pid)
-          processes[pid] << log
+        if @processes.has_key?(pid)
+          @processes[pid] << log
           if end_action?(@message[log])
-            output.call(parse_action_logs(processes.delete(pid)))
+            parse_action_logs(@processes.delete(pid))
           end
         else
           if start_action?(@message[log])
-            processes[pid] = [log]
-          else
-            @unmatched.call(log)
+            @processes[pid] = [log]
           end
+          nil
         end
       end
     end

data/lib/evidence.rb

@@ -1,51 +1,70 @@
-require 'evidence/stream'
-require 'evidence/log_parser'
+require 'evidence/lazy'
 require 'evidence/rails_action_parser'
 
+Enumerator::Lazy.send(:include, Evidence::Lazy)
+
 module Evidence
   module_function
 
   # Parse log file stream by given pattern
   #   pattern: ruby regex expression, has named group specified
-  #   unmatched processor: process all unmatched log
-  def log_parser(pattern, unmatched=default_unmatched_process)
-    LogParser.new(pattern, unmatched)
+  # example: logs.map(&parse_log(pattern)).compact
+  def parse_log(pattern)
+    lambda do |log|
+      if m = pattern.match(log)
+        Hash[m.names.map(&:to_sym).zip(m.captures)]
+      end
+    end
   end
 
   # Parse out rails actions by given:
   #   pid: a lambda returns process id used to group logs
   #   message: a lambda returns rails log string message
-  def rails_action_parser(pid, message, unmatched=default_unmatched_process)
-    RailsActionParser.new(pid, message, unmatched)
+  # example: logs.map(&rails_action_parser(pid, message)).compact
+  def rails_action_parser(pid, message)
+    RailsActionParser.new(pid, message)
   end
 
   # Rails action request timestamp parser
-  #   log stream | rails_action_parser(pid, message) | request_timestamp_parser
+  #   log.map(&rails_action_parser(pid, message)).compact.map(&request_timestamp_parser)
   def request_timestamp_parser(format="%Y-%m-%d %H:%M:%S")
-    stream_each do |action|
+    lambda do |action|
       action[:request][:timestamp] = Time.strptime(action[:request][:timestamp], format)
+      action
     end
   end
 
-  # Do the little's law analysis on rails actions stream with request_timestamp_parser
-  # usage example:
-  #   log stream | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, 60) | littles_law_analysis
-  def littles_law_analysis
-    lambda do |output|
-      lambda do |actions|
-        statistics = actions[:stream].inject(sum: 0, count: 0) do |memo, action|
-          memo[:count] += 1
-          memo[:sum] += action[:response][:completed_time].to_i
-          memo
-        end
-        avg_sec_arrival_rate = statistics[:count].to_f/(actions[:range].max - actions[:range].min)
-        avg_sec_response_time = statistics[:sum].to_f / statistics[:count] /1000
-        output[range: actions[:range], value: avg_sec_arrival_rate * avg_sec_response_time]
+  # actions.chunk(&time_window(60))
+  def by_time_window(time_window, start=nil)
+    range = nil
+    lambda do |ele|
+      start ||= ele[:request][:timestamp]
+      range ||= start..(start + time_window)
+      while(range.max <= ele[:request][:timestamp]) do
+        range = range.max..(range.max + time_window)
       end
+      range
     end
   end
 
-  def default_unmatched_process
-    lambda {|log| }
+  # Do the little's law analysis on rails actions stream
+  # usage example:
+  #   rails_action_parser(pid, message).parse(parse_log(logs,
+  # pattern)).each(&request_timestamp_parser).chunk({start: nil},
+  # &time_window(60 seconds)).map do |range, actions|
+  #     littles_law_analysis(range, actions)
+  #   end
+  def littles_law_analysis
+    lambda do |args|
+      range, actions = args
+      statistics = actions.inject(sum: 0, count: 0) do |memo, action|
+        memo[:count] += 1
+        memo[:sum] += action[:response][:completed_time].to_i
+        memo
+      end
+      avg_sec_arrival_rate = statistics[:count].to_f/(range.max - range.min)
+      avg_sec_response_time = statistics[:sum].to_f / statistics[:count] /1000
+      {range: range, value: avg_sec_arrival_rate * avg_sec_response_time}
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: evidence
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Xiao Li
@@ -9,20 +9,20 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-14 00:00:00.000000000 Z
+date: 2013-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -34,9 +34,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - README.md
-- lib/evidence/log_parser.rb
+- lib/evidence/lazy.rb
 - lib/evidence/rails_action_parser.rb
-- lib/evidence/stream.rb
 - lib/evidence.rb
 - examples/mingle_logs_analysis.rb
 homepage: https://github.com/ThoughtWorksStudios/evidence
@@ -49,17 +48,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
 summary: Log Analysis Tool

data/lib/evidence/log_parser.rb

@@ -1,21 +0,0 @@
-module Evidence
-  class LogParser
-    def initialize(pattern, unmatched)
-      @pattern, @unmatched = pattern, unmatched
-    end
-
-    def [](output)
-      lambda do |line|
-        if m = @pattern.match(line)
-          output.call(to_hash(m))
-        else
-          @unmatched.call(line)
-        end
-      end
-    end
-
-    def to_hash(m)
-      Hash[m.names.map(&:to_sym).zip(m.captures)]
-    end
-  end
-end

data/lib/evidence/stream.rb

@@ -1,195 +0,0 @@
-module Evidence
-  module Stream
-    include Enumerable
-
-    def |(process)
-      case process
-      when SliceStream
-        process.slice(self)
-      else
-        PipeStream.new(self, process)
-      end
-    end
-  end
-
-  class PipeStream
-    include Stream
-
-    def initialize(upstream, process)
-      @upstream, @process = upstream, process
-    end
-
-    def eos?
-      @upstream.eos?
-    end
-
-    def each(&output)
-      @upstream.each(&@process[output])
-    end
-  end
-
-  class EnumStream
-    include Stream
-
-    def initialize(enum)
-      @enum = enum
-    end
-
-    def eos?
-      @enum.peek
-      false
-    rescue StopIteration
-      true
-    end
-
-    def each(&output)
-      loop do
-        output[@enum.next]
-      end
-    rescue StopIteration
-    end
-
-    def to_s
-      "$[#{@enum.inspect}]"
-    end
-  end
-
-  class MergedStream
-    include Stream
-
-    def initialize(streams, comparator)
-      @comparator = comparator
-      @heads = streams.map{|s| {stream: s}}
-    end
-
-    def eos?
-      @heads.empty?
-    end
-
-    def each(&output)
-      pull_heads
-      loop do
-        if min = @heads.min{|a, b| @comparator.call(a[:element], b[:element])}
-          output.call(min.delete(:element).tap{ pull_heads })
-        else
-          break if @heads.empty?
-          pull_heads
-        end
-      end
-    end
-
-    def pull_heads
-      @heads.select!{|h| h[:element] ||= pull(h[:stream])}
-    end
-
-    def pull(stream)
-      loop do
-        return nil if stream.eos?
-        if n = stream.first
-          return n
-        end
-      end
-    end
-  end
-
-  class SlicedStreams
-    include Stream
-
-    def initialize(stream, index, start_index, end_index)
-      @stream, @index, @start_index, @end_index = stream, index, start_index, end_index
-      @slice_start_index, @slice_end_index = nil
-    end
-
-    def eos?
-      @stream.eos?
-    end
-
-    def each(&output)
-      return if eos?
-      @head ||= @stream.first
-      @slice_start_index ||= @start_index || @index[@head]
-      @slice_end_index ||= @end_index[@slice_start_index]
-      @eos_in_slice ||= false
-      loop do
-        if @slice_start_index > @index[@head]
-          return if eos?
-          @head = @stream.first
-        else
-          break
-        end
-      end
-
-      loop do
-        break if @eos_in_slice
-        range = @slice_start_index..@slice_end_index
-        slice_enum = Enumerator.new do |y|
-          loop do
-            break if range.max <= @index[@head]
-            if @eos_in_slice = eos?
-              y << @head
-              break
-            end
-            head, @head = @head, @stream.first
-            y << head
-          end
-        end
-        @slice_start_index, @slice_end_index = range.max, @end_index[range.max]
-        output[range: range, stream: EnumStream.new(slice_enum)]
-      end
-    end
-  end
-
-  class SliceStream
-    def initialize(index, start_index, end_index)
-      @index, @start_index, @end_index = index, start_index, end_index
-    end
-
-    def slice(stream)
-      SlicedStreams.new(stream, @index, @start_index, @end_index)
-    end
-  end
-
-  module_function
-  def stream(obj)
-    EnumStream.new(obj.to_enum)
-  end
-
-  def merge_streams(streams, comparator)
-    loop do
-      s1 = streams.shift
-      return s1 if streams.empty?
-      s2 = streams.shift
-      streams << MergedStream.new([s1, s2], comparator)
-    end
-  end
-
-  def slice_stream(index, step, start_index=nil)
-    end_index = step.is_a?(Proc) ? step : lambda { |index| index + step }
-    SliceStream.new(index, start_index, end_index)
-  end
-
-  def stream_each(&block)
-    lambda do |output|
-      lambda do |i|
-        block[i]
-        output[i]
-      end
-    end
-  end
-
-  def stream_map(&block)
-    lambda { |output| lambda { |i| output[block[i]] } }
-  end
-
-  def stream_filter(&block)
-    lambda { |output| lambda { |i| output[i] if block[i] } }
-  end
-  alias :stream_select :stream_filter
-
-  def counter
-    count = 0
-    counter = Enumerator.new { |y| loop { y << (count += 1) } }
-    stream(counter)
-  end
-
-end