josephruscio-aggregate 0.0.1

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2009 Joseph Ruscio
+ 
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/aggregate.rb

@@ -0,0 +1,240 @@
+# Implements aggregate statistics and maintains
+# configurable histogram for a set of given samples. Convenient for tracking
+# high throughput data.
+class Aggregate
+  #The current average of all samples
+  attr_reader :mean
+
+  #The current number of samples 
+  attr_reader :count
+  
+  #The maximum sample value
+  attr_reader :max
+  
+  #The minimum samples value
+  attr_reader :min
+  
+  #The sum of all samples
+  attr_reader :sum
+
+  #The number of samples falling below the lowest valued histogram bucket
+  attr_reader :outliers_low
+
+  #The number of samples falling above the highest valued histogram bucket
+  attr_reader :outliers_high
+ 
+  # The number of buckets in the binary logarithmic histogram (low => 2**0, high => 2**@@LOG_BUCKETS)
+  @@LOG_BUCKETS = 128
+
+  # Create a new Aggregate that maintains a binary logarithmic histogram
+  # by default. Specifying values for low, high, and width configures
+  # the aggregate to maintain a linear histogram with (high - low)/width buckets
+  def initialize (low=nil, high=nil, width=nil)
+    @count = 0
+    @sum = 0.0
+    @sum2 = 0.0
+    @outliers_low = 0
+    @outliers_high = 0
+
+    # If the user asks we maintain a linear histogram
+    # STILL UNDER TEST/DEV
+    if false #(nil != low && nil != high && nil != width)
+      # This is a linear histogram
+      if high < low
+	raise ArgumentError, "High bucket must be > Low bucket"
+      end
+
+      @low = low
+      @high = high
+      @width = width
+    else
+      @low = 1
+      @high = to_bucket(@@LOG_BUCKETS - 1)
+    end
+
+    #Initialize all buckets to 0
+    @buckets = Array.new(bucket_count, 0)
+  end
+
+  # Include a sample in the aggregate
+  def << data
+
+    # Update min/max
+    if 0 == @count
+      @min = data
+      @max = data
+    elsif data > @max
+      @max = data
+    elsif data < @min
+      @min = data
+    end
+
+    # Update the running info
+    @count += 1 
+    @sum += data
+    @sum2 += (data * data)
+
+    # Update the bucket
+    @buckets[to_index(data)] += 1 unless outlier?(data)
+  end
+
+  def mean
+    @sum / self.count
+  end
+
+  def std_dev
+  end
+
+  #Generate a pretty-printed ASCII representation of the histogram
+  def to_s
+    #Find the largest bucket and create an array of the rows we intend to print
+    max_count = 0
+    disp_buckets = Array.new
+    @buckets.each_with_index do |count, idx|
+      next if 0 == count
+      max_count = count if max_count < count
+      disp_buckets << [idx, to_bucket(idx), count]
+    end
+
+    #Figure out how wide the value and count columns need to be based on their
+    #largest respective numbers
+    value_width = [disp_buckets.last[1].to_s.length, "value".length].max
+    count_width = [max_count.to_s.length, "count".length].max
+    max_bar_width  = 80 - (value_width + " |".length + " ".length + count_width)
+
+    #print the header
+    header = sprintf("%#{value_width}s", "value")
+    header += " |"
+    max_bar_width.times { header += "-"}
+    header += " count"
+
+    #Determine the value of a '@'
+    weight = [max_count.to_f/max_bar_width.to_f, 1.0].max
+
+    #Loop through each bucket to be displayed and output the correct number
+    histogram = ""
+    prev_index = disp_buckets[0][0] - 1
+    disp_buckets.each do |x|
+
+      #Denote skipped empty buckets with a ~
+      histogram += "      ~\n" unless prev_index == x[0] - 1
+      prev_index = x[0]
+
+      #Add the value
+      row = sprintf("%#{value_width}d |", x[1])
+
+      #Add the bar
+      bar_size = (x[2]/weight).to_i
+      bar_size.times { row += "@"}
+      (max_bar_width - bar_size).times { row += " " }
+
+      #Add the count
+      row += sprintf(" %#{count_width}d\n", x[2])
+
+      #Append the finished row onto the histogram
+      histogram += row
+    end
+
+    #Put the pieces together
+    "\n" + header + "\n" + histogram 
+  end
+ 
+  #Iterate through each bucket in the histogram regardless of 
+  #its contents 
+  def each
+    @buckets.each_with_index do |count, index|
+      yield(to_bucket(index), count)
+    end
+  end
+
+  #Iterate through only the buckets in the histogram that contain
+  #samples
+  def each_nonzero
+    @buckets.each_with_index do |count, index|
+      yield(to_bucket(index), count) if count != 0
+    end
+  end
+
+  private
+
+  def linear?
+    nil != @width
+  end
+
+  def outlier? (data)
+
+    if data < @low
+      @outliers_low += 1
+    elsif data > @high
+      @outliers_high += 1
+    else
+      return false
+    end
+  end
+
+  def bucket_count
+    if linear?
+      return (@high-@low)/@width
+    else
+      return @@LOG_BUCKETS
+    end
+  end
+
+  def to_bucket(index)
+    if linear?
+      return @low + ( (index + 1) * @width)
+    else
+      return 2**(index)
+    end
+  end
+    
+  def right_bucket?(index, data)
+    bucket = to_bucket(index)
+
+    # Edge case
+    if 0 == index
+      prev_bucket = @low
+    else
+      prev_bucket = to_bucket(index - 1)
+    end
+
+    #It's the right bucket if data falls between prev_bucket and bucket
+    prev_bucket <= data && data <= bucket
+  end
+
+  def find_bucket(lower, upper, target)
+    #Classic binary search
+    return upper if right_bucket?(upper, target)
+
+    # Cut the search range in half
+    middle = (upper/2).to_i
+
+    # Determine which half contains our value and recurse
+    if (to_bucket(middle) >= target)
+      return find_bucket(lower, middle, target)
+    else
+      return find_bucket(middle, upper, target)
+    end
+  end
+
+  # A data point is added to the bucket[n] where the data point
+  # is less than the value represented by bucket[n], but greater
+  # than the value represented by bucket[n+1]
+  def to_index (data)
+
+    if linear?
+      find_bucket(0, bucket_count-1, data)
+    else
+      #log2 returns the bucket above the one we want,
+      #and we need to also subtract for 0 indexing of Array
+      log2(data).to_i
+    end
+
+  end
+
+  # log2(x) returns j, | i = j-1 and 2**i <= data < 2**j
+  def log2( x )
+   Math.log(x) / Math.log(2)
+  end
+ 
+end

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification 
+name: josephruscio-aggregate
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Joseph Ruscio
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-11 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Aggregate is a Ruby library accumulating aggregate statistics (including histograms) in an object oriented manner.
+email: jruscio@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+files: 
+- aggregate.rb
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/josephruscio/aggregate
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: Aggregate is a Ruby library accumulating aggregate statistics (including histograms) in an object oriented manner.
+test_files: []
+