evidence 0.0.1

data/examples/mingle_logs_analysis.rb

@@ -0,0 +1,66 @@
+require 'evidence'
+require 'time'
+
+include Evidence
+
+mingle_log_pattern = /^
+  \w{3}\s+\d+\s+\d{2}\:\d{2}\:\d{2}\s+
+  (?<host_name>[^\s]+)\s+
+  [\w-_]+\:\s+
+  INFO\s+
+  \[(?<timestamp>[^\]]+)\]\s+
+  \[(?<thread_label>[^\]]+)\]\s+
+  \[(?<log4j_label>[^\]]+)\]\s+
+  \[tenant\:(?<tenant>[^\]]*)\]\s+
+  (?<message>.*)
+$/x
+mingle_bg_log_pattern = /^
+  \w{3}\s+\d+\s+\d{2}\:\d{2}\:\d{2}\s+
+  (?<host_name>[^\s]+)\s+
+  [\w-_]+\:\s+
+  INFO\s+
+  \[(?<timestamp>[^\]]+)\]\s+
+  \[[\w-_\d]+\[(?<thread_label>[^\]]+)\]\]\s+
+  \[(?<log4j_label>[^\]]+)\]\s+
+  \[tenant\:(?<tenant>[^\]]*)\]\s+
+  (?<message>.*)
+$/x
+
+pid = lambda {|log| "#{log[:host_name]}-#{log[:thread_label]}"}
+message = lambda {|log| log[:message]}
+
+def parse_timestamp
+  lambda do |block|
+    lambda do |log|
+      block.call(log.merge(timestamp: Time.parse(log[:timestamp])))
+    end
+  end
+end
+
+logs = Dir['./../mingle-saas/log/dumpling/**/mingle-cluster*'].reject do |f|
+  File.new(f).first(100).any? do |l|
+    l =~ mingle_bg_log_pattern
+  end
+end.map do |f|
+  stream(File.new(f), log_parser(mingle_log_pattern))
+end
+puts "[DEBUG]logs.size => #{logs.size.inspect}"
+
+merged = merge_streams(logs, lambda {|log1, log2| log1[:timestamp] <=> log2[:timestamp]})
+
+#counter = 0
+
+#stream(merged, lambda {|block| lambda {|log| block.call(counter += 1)}}).each do |count|
+#   puts "#{Time.now}: #{count}"
+#end
+
+actions_stream = stream(merged, rails_action_parser(pid, message))
+
+time_window = (ARGV[0] || 60).to_i
+File.open('datafile', 'w') do |f|
+  stream(actions_stream, littles_law_analysis(time_window)).map do |start_time, end_time, avg_stay_in_system|
+    t1 = start_time.strftime("%m-%d %H:%M")
+    t2 = end_time.strftime("%H:%M")
+    f.write("#{t1}-#{t2} #{avg_stay_in_system}\n".tap{|r| puts r})
+  end
+end

data/lib/evidence/littles_law_analysis.rb

@@ -0,0 +1,56 @@
+require 'time'
+
+module Evidence
+  # Little's Law analysis, input stream log should include the following data
+  # {:request => {:timestamp}, :response => {:completed_time}}
+  class LittlesLawAnalysis
+    class Result
+      def initialize(start, time_window)
+        @start, @time_window = start, time_window
+        @end = @start + @time_window
+        @arrival_count = 0
+        @response_time = 0
+      end
+
+      def ended?(timestamp)
+        @end <= timestamp
+      end
+
+      def add(millisecond)
+        @arrival_count += 1
+        @response_time += millisecond
+      end
+
+      def next
+        Result.new(@start + @time_window, @time_window)
+      end
+
+      def value
+        avg_sec_arrival_rate = @arrival_count.to_f/@time_window
+        avg_sec_response_time = @response_time.to_f/1000/@arrival_count
+        [@start, @end, avg_sec_arrival_rate * avg_sec_response_time]
+      end
+    end
+
+    # time_window: second
+    def initialize(time_window)
+      @time_window = time_window
+    end
+
+    def [](block)
+      result = nil
+      lambda do |log|
+        timestamp = Time.parse(log[:request][:timestamp])
+        if result.nil?
+          result = Result.new(timestamp, @time_window)
+        else
+          if result.ended?(timestamp)
+            block.call(result.value)
+            result = result.next
+          end
+        end
+        result.add(log[:response][:completed_time].to_i)
+      end
+    end
+  end
+end

data/lib/evidence/log_parser.rb

@@ -0,0 +1,25 @@
+module Evidence
+  class LogParser
+    def initialize(pattern, unmatched)
+      @pattern, @unmatched = pattern, unmatched
+    end
+
+    def [](block)
+      single_pattern_parser(block)
+    end
+
+    def single_pattern_parser(block)
+      lambda do |line|
+        if m = @pattern.match(line)
+          block.call(to_hash(m))
+        else
+          @unmatched.call(line)
+        end
+      end
+    end
+
+    def to_hash(m)
+      Hash[m.names.map(&:to_sym).zip(m.captures)]
+    end
+  end
+end

data/lib/evidence/rails_action_parser.rb

@@ -0,0 +1,83 @@
+module Evidence
+  class RailsActionParser
+
+    def initialize(pid, message, unmatched)
+      @pid, @message = pid, message
+      @unmatched = unmatched
+    end
+
+    def [](block)
+      processes = Hash.new
+      lambda do |log|
+        pid = @pid[log]
+        if processes.has_key?(pid)
+          processes[pid] << log
+          if end_action?(@message[log])
+            block.call(parse_action_logs(processes.delete(pid)))
+          end
+        else
+          if start_action?(@message[log])
+            processes[pid] = [log]
+          else
+            @unmatched.call(log)
+          end
+        end
+      end
+    end
+
+    def parse_action_logs(logs)
+      {
+        request: request(@message[logs[0]]),
+        response: response(@message[logs[-1]]),
+        logs: logs
+      }
+    end
+
+    def end_action?(msg)
+      msg =~ end_action_pattern
+    end
+
+    def start_action?(msg)
+      msg =~ start_action_pattern
+    end
+
+    def request(msg)
+      to_hash(start_action_pattern.match(msg))
+    end
+
+    def response(msg)
+      to_hash(end_action_pattern.match(msg))
+    end
+
+    def start_action_pattern
+      /^
+        (\#012\#012)?             # ignore encoded newlines
+        Processing\s+
+        (?<controller>\w+)\#(?<action>\w+)\s+
+        \(for\s+
+        (?<remote_addr>[^\s]+)\s+
+        at\s+
+        (?<timestamp>[^\)]+)\)\s+
+        \[(?<method>[\w]+)\]
+      $/x
+    end
+
+    # Completed in 755ms (View: 330, DB: 215) | 200 OK [url]
+    def end_action_pattern
+      /^
+        Completed\sin\s
+        (?<completed_time>\d+)ms\s+
+        \(View\:\s(?<view_time>\d+)
+        (,\s*DB\:\s(?<db_time>\d+))?
+        \)\s+\|\s+
+        (?<code>\d+)\s+
+        (?<status>\w+)\s+
+        \[(?<url>.+)\]
+      $/x
+    end
+
+    def to_hash(m)
+      Hash[m.names.map(&:to_sym).zip(m.captures)]
+    end
+  end
+end

data/lib/evidence/stream.rb

@@ -0,0 +1,109 @@
+module Evidence
+  class Stream
+    include Enumerable
+
+    def initialize(upstream, processor)
+      @upstream, @processor = upstream, processor
+    end
+
+    def eos?
+      @upstream.eos?
+    end
+
+    def each(&block)
+      @upstream.each(&@processor[block])
+    end
+  end
+
+  class FileStream
+    include Enumerable
+
+    def initialize(file)
+      @file = file
+    end
+
+    def eos?
+      @file.eof?
+    end
+
+    def each(&block)
+      @file.each(&block)
+    end
+  end
+
+  class ArrayStream
+    include Enumerable
+    def initialize(array)
+      @array = array
+    end
+
+    def eos?
+      @array.empty?
+    end
+
+    def each(&block)
+      while(item = @array.shift) do
+        block.call(item)
+      end
+    end
+
+    def to_s
+      "$[#{@array.inspect}]"
+    end
+  end
+
+  class MergedStream
+    include Enumerable
+
+    def initialize(streams, comparator)
+      @comparator = comparator
+      @heads = streams.map{|s| {stream: s}}
+    end
+
+    def eos?
+      @heads.empty?
+    end
+
+    def each(&block)
+      pull_heads
+      loop do
+        if min = @heads.min{|a, b| @comparator.call(a[:element], b[:element])}
+          block.call(min.delete(:element).tap{ pull_heads })
+        else
+          break if @heads.empty?
+          pull_heads
+        end
+      end
+    end
+
+    def pull_heads
+      @heads.select!{|h| h[:element] ||= pull(h[:stream])}
+    end
+
+    def pull(stream)
+      loop do
+        return nil if stream.eos?
+        if n = stream.first
+          return n
+        end
+      end
+    end
+  end
+
+  class Counter
+    include Enumerable
+    def initialize
+      @count = 0
+    end
+
+    def eos?
+      false
+    end
+
+    def each(&block)
+      loop do
+        block.call(@count += 1)
+      end
+    end
+  end
+end

data/lib/evidence.rb

@@ -0,0 +1,51 @@
+require 'evidence/stream'
+require 'evidence/log_parser'
+require 'evidence/rails_action_parser'
+require 'evidence/littles_law_analysis'
+
+module Evidence
+  module_function
+
+  # A stream is an Enumerable with a processor processing the data comming
+  # from upstream and yield to downstream
+  def stream(obj, processor=nil)
+    up_stream = case obj
+    when Array
+      ArrayStream.new(obj)
+    when File
+      FileStream.new(obj)
+    else
+      obj
+    end
+    Stream.new(up_stream, processor || lambda {|b| b})
+  end
+
+  def merge_streams(streams, comparator)
+    loop do
+      s1 = streams.shift
+      return s1 if streams.empty?
+      s2 = streams.shift
+      streams << MergedStream.new([s1, s2], comparator)
+    end
+  end
+
+  def counter
+    Counter.new
+  end
+
+  def log_parser(pattern, unmatched=default_unmatched_processor)
+    LogParser.new(pattern, unmatched)
+  end
+
+  def rails_action_parser(pid, message, unmatched=default_unmatched_processor)
+    RailsActionParser.new(pid, message, unmatched)
+  end
+
+  def littles_law_analysis(time_window)
+    LittlesLawAnalysis.new(time_window)
+  end
+
+  def default_unmatched_processor
+    lambda {|log| }
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: evidence
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Xiao Li
+- Sheroy Marker
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- swing1979@gmail.com
+- smarker@thoughtworks.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/evidence/littles_law_analysis.rb
+- lib/evidence/log_parser.rb
+- lib/evidence/rails_action_parser.rb
+- lib/evidence/stream.rb
+- lib/evidence.rb
+- examples/mingle_logs_analysis.rb
+homepage: https://github.com/ThoughtWorksStudios/evidence
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Log Analysis Tool
+test_files: []