evidence 0.0.2 → 0.0.3

data/lib/evidence/log_parser.rb

@@ -5,10 +5,6 @@ module Evidence
     end
 
     def [](output)
-      single_pattern_parser(output)
-    end
-
-    def single_pattern_parser(output)
       lambda do |line|
         if m = @pattern.match(line)
           output.call(to_hash(m))

data/lib/evidence/stream.rb

@@ -1,15 +1,19 @@
 module Evidence
-  module Pipeline
+  module Stream
+    include Enumerable
+
     def |(process)
-      Stream.new(self, process)
+      case process
+      when SliceStream
+        process.slice(self)
+      else
+        PipeStream.new(self, process)
+      end
     end
   end
 
-  # A stream is an Enumerable with a process processing the data comming
-  # from input stream and output as another stream
-  class Stream
-    include Enumerable
-    include Pipeline
+  class PipeStream
+    include Stream
 
     def initialize(upstream, process)
       @upstream, @process = upstream, process
@@ -24,49 +28,34 @@ module Evidence
     end
   end
 
-  class FileStream
-    include Enumerable
-    include Pipeline
+  class EnumStream
+    include Stream
 
-    def initialize(file)
-      @file = file
+    def initialize(enum)
+      @enum = enum
     end
 
     def eos?
-      @file.eof?
-    end
-
-    def each(&output)
-      @file.each(&output)
-    end
-  end
-
-  class ArrayStream
-    include Enumerable
-    include Pipeline
-
-    def initialize(array)
-      @array = array
-    end
-
-    def eos?
-      @array.empty?
+      @enum.peek
+      false
+    rescue StopIteration
+      true
     end
 
     def each(&output)
-      while(item = @array.shift) do
-        output.call(item)
+      loop do
+        output[@enum.next]
       end
+    rescue StopIteration
     end
 
     def to_s
-      "$[#{@array.inspect}]"
+      "$[#{@enum.inspect}]"
     end
   end
 
   class MergedStream
-    include Enumerable
-    include Pipeline
+    include Stream
 
     def initialize(streams, comparator)
       @comparator = comparator
@@ -103,22 +92,104 @@ module Evidence
     end
   end
 
-  class Counter
-    include Enumerable
-    include Pipeline
+  class SlicedStreams
+    include Stream
 
-    def initialize
-      @count = 0
+    def initialize(stream, index, start_index, end_index)
+      @stream, @index, @start_index, @end_index = stream, index, start_index, end_index
+      @slice_start_index, @slice_end_index = nil
     end
 
     def eos?
-      false
+      @stream.eos?
     end
 
     def each(&output)
+      return if eos?
+      @head ||= @stream.first
+      @slice_start_index ||= @start_index || @index[@head]
+      @slice_end_index ||= @end_index[@slice_start_index]
+      @eos_in_slice ||= false
+      loop do
+        if @slice_start_index > @index[@head]
+          return if eos?
+          @head = @stream.first
+        else
+          break
+        end
+      end
+
       loop do
-        output.call(@count += 1)
+        break if @eos_in_slice
+        range = @slice_start_index..@slice_end_index
+        slice_enum = Enumerator.new do |y|
+          loop do
+            break if range.max <= @index[@head]
+            if @eos_in_slice = eos?
+              y << @head
+              break
+            end
+            head, @head = @head, @stream.first
+            y << head
+          end
+        end
+        @slice_start_index, @slice_end_index = range.max, @end_index[range.max]
+        output[range, EnumStream.new(slice_enum)]
       end
     end
   end
+
+  class SliceStream
+    def initialize(index, start_index, end_index)
+      @index, @start_index, @end_index = index, start_index, end_index
+    end
+
+    def slice(stream)
+      SlicedStreams.new(stream, @index, @start_index, @end_index)
+    end
+  end
+
+  module_function
+  def stream(obj)
+    EnumStream.new(obj.to_enum)
+  end
+
+  def merge_streams(streams, comparator)
+    loop do
+      s1 = streams.shift
+      return s1 if streams.empty?
+      s2 = streams.shift
+      streams << MergedStream.new([s1, s2], comparator)
+    end
+  end
+
+  def slice_stream(index, step, start_index=nil)
+    end_index = step.is_a?(Proc) ? step : lambda { |index| index + step }
+    SliceStream.new(index, start_index, end_index)
+  end
+
+  def stream_each(&block)
+    lambda do |output|
+      lambda do |i|
+        block[i]
+        output[i]
+      end
+    end
+  end
+
+  def stream_map(&block)
+    lambda { |output| lambda { |i| output[block[i]] } }
+  end
+
+  def stream_filter(&block)
+    lambda { |output| lambda { |i| output[i] if block[i] } }
+  end
+  alias :stream_select :stream_filter
+
+  def counter
+    count = 0
+    counter = Enumerator.new { |y| loop { y << (count += 1) } }
+    stream(counter)
+  end
+
 end

data/lib/evidence.rb

@@ -5,53 +5,9 @@ require 'evidence/rails_action_parser'
 module Evidence
   module_function
 
-  # convert Array or File object to a stream
-  def stream(obj)
-    case obj
-    when Array
-      ArrayStream.new(obj)
-    when File
-      FileStream.new(obj)
-    else
-      raise "Unknown how to convert #{obj.class} to a stream"
-    end
-  end
-
-  def merge_streams(streams, comparator)
-    loop do
-      s1 = streams.shift
-      return s1 if streams.empty?
-      s2 = streams.shift
-      streams << MergedStream.new([s1, s2], comparator)
-    end
-  end
-
-  def slice_stream(index, step, start_index=nil)
-    end_index = step.is_a?(Proc) ? step : lambda { |index| index + step }
-    lambda do |output|
-      @cache ||= []
-      start_index ||= @cache.first ? index[@cache.first] : nil
-      lambda do |log|
-        next_index = index[log]
-        start_index = index[log] if start_index.nil?
-        return if start_index > next_index
-        @cache << log
-        if end_index[start_index] <= next_index
-          range = start_index..end_index[start_index]
-          start_index = range.max
-          output.call(range, @cache.shift(@cache.size - 1))
-        end
-      end
-    end
-  end
-
-  def counter
-    Counter.new
-  end
-
   # Parse log file stream by given pattern
   #   pattern: ruby regex expression, has named group specified
-  #   output stream: hash object with name and captured string in log
+  #   unmatched processor: process all unmatched log
   def log_parser(pattern, unmatched=default_unmatched_process)
     LogParser.new(pattern, unmatched)
   end
@@ -66,11 +22,8 @@ module Evidence
   # Rails action request timestamp parser
   #   log stream | rails_action_parser(pid, message) | request_timestamp_parser
   def request_timestamp_parser(format="%Y-%m-%d %H:%M:%S")
-    lambda do |output|
-      lambda do |action|
-        action[:request][:timestamp] = Time.strptime(action[:request][:timestamp], format)
-        output[action]
-      end
+    stream_each do |action|
+      action[:request][:timestamp] = Time.strptime(action[:request][:timestamp], format)
     end
   end
 
@@ -79,12 +32,14 @@ module Evidence
   #   log stream | rails_action_parser(pid, message) | request_timestamp_parser | slice_stream(lambda {|action| action[:request][:timestamp]}, 60) | littles_law_analysis
   def littles_law_analysis
     lambda do |output|
-      lambda do |range, logs|
-        count = logs.size
-        avg_response_time = logs.reduce(0) {|memo, log| memo + log[:response][:completed_time].to_i} / count
-
-        avg_sec_arrival_rate = count.to_f/(range.max - range.min)
-        avg_sec_response_time = avg_response_time.to_f/1000
+      lambda do |range, actions|
+        statistics = actions.inject(sum: 0, count: 0) do |memo, action|
+          memo[:count] += 1
+          memo[:sum] += action[:response][:completed_time].to_i
+          memo
+        end
+        avg_sec_arrival_rate = statistics[:count].to_f/(range.max - range.min)
+        avg_sec_response_time = statistics[:sum].to_f / statistics[:count] /1000
         output[range, avg_sec_arrival_rate * avg_sec_response_time]
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: evidence
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-11 00:00:00.000000000 Z
+date: 2013-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake