wonkavision 0.5.11 → 0.6.0

data/lib/wonkavision/plugins.rb

@@ -4,6 +4,7 @@
 #names to avoid conflicts.
 module Wonkavision
   module Plugins
+
     def wonkavision_plugins
       @wonkavision_plugins ||= []
     end
@@ -25,6 +26,6 @@ module Wonkavision
       wonkavision_plugins << mod
     end
     alias use plug
-    
+
   end
 end

data/lib/wonkavision/plugins/analytics/aggregation.rb

@@ -0,0 +1,139 @@
+module Wonkavision
+  module Plugins
+    module Aggregation
+
+      def self.all
+        @@all ||= {}
+      end
+
+      def self.configure(aggregation,options={})
+        aggregation.write_inheritable_attribute :aggregation_options, options
+        aggregation.class_inheritable_reader :aggregation_options
+
+        aggregation.write_inheritable_attribute( :aggregation_spec,
+                                                 AggregationSpec.new(aggregation.name) )
+        aggregation.class_inheritable_reader :aggregation_spec
+
+        Aggregation.all[aggregation.name] = aggregation
+      end
+
+      module ClassMethods
+        def store(new_store=nil)
+          if new_store
+            store = new_store.kind_of?(Wonkavision::Analytics::Persistence::Store) ? store :
+              Wonkavision::Analytics::Persistence::Store[new_store]
+
+            raise "Could not find a storage type of #{new_store}" unless store
+
+            store = store.new(self) if store.respond_to?(:new)
+
+            aggregation_options[:store] = store
+          else
+            aggregation_options[:store]
+          end
+        end
+
+
+        def [](dimensions)
+          key = [dimension_names(dimensions),dimension_keys(dimensions)]
+          @instances ||= HashWithIndifferentAccess.new
+          @instances[key] ||= self.new(dimensions)
+        end
+
+        def aggregates(facts_class = nil)
+          return aggregation_options[:facts_class] unless facts_class
+
+          facts_class.aggregations << self
+          aggregation_options[:facts_class] = facts_class
+        end
+        alias facts aggregates
+
+        def dimension_names(dimensions)
+          dimensions.keys.sort
+        end
+
+        def dimension_keys(dimensions)
+          dimension_names(dimensions).map do |dim|
+            dimensions[dim][self.dimensions[dim].key.to_s]
+          end
+        end
+
+        def query(options={},&block)
+          raise "Aggregation#query is not valid unless a store has been configured" unless store
+          query = Wonkavision::Analytics::Query.new
+          query.instance_eval(&block) if block
+          query.validate!
+
+          return query if options[:defer]
+
+          tuples = store.execute_query(query)
+
+          Wonkavision::Analytics::CellSet.new( self,
+                                               query,
+                                               tuples )
+        end
+
+
+        def method_missing(m,*args,&block)
+          aggregation_spec.respond_to?(m) ? aggregation_spec.send(m,*args,&block) : super
+        end
+      end
+
+      module InstanceMethods
+        attr_reader :dimensions, :measures
+
+        def initialize(dimensions)
+          @dimensions = dimensions
+        end
+
+        def add(measures)
+          update(measures, :add)
+        end
+
+        def reject(measures)
+          update(measures, :reject)
+        end
+
+        def dimension_names
+          @dimension_names ||= self.class.dimension_names(@dimensions)
+        end
+
+        def dimension_keys
+          @dimension_keys ||= self.class.dimension_keys(@dimensions)
+        end
+
+        protected
+        def update(measures, method)
+          aggregation = {
+            :dimension_keys => dimension_keys,
+            :dimension_names => dimension_names,
+            :measures => {},
+            :dimensions => @dimensions
+          }
+
+          measures.keys.each do |measure|
+            if val = measures[measure]
+              aggregation[:measures].merge! measure_changes_for(measure.to_s,
+                                                                val,
+                                                                method)
+            end
+
+          end
+          self.class.store.update_aggregation(aggregation)
+          self
+        end
+
+        def measure_changes_for(measure_name, measure_value, update_method)
+          sign = update_method.to_s == "reject" ? -1 : 1
+          {
+            "measures.#{measure_name}.count" => 1 * sign,
+            "measures.#{measure_name}.sum" => measure_value * sign,
+            "measures.#{measure_name}.sum2" => (measure_value * measure_value) * sign
+          }
+        end
+
+      end
+
+    end
+  end
+end

data/lib/wonkavision/plugins/analytics/aggregation/aggregation_spec.rb

@@ -0,0 +1,53 @@
+module Wonkavision
+  module Plugins
+    module Aggregation
+      class AggregationSpec
+
+        attr_reader :name, :dimensions, :measures, :aggregations, :filter
+
+        def initialize(name)
+          @name = name
+          @measures = HashWithIndifferentAccess.new
+          @aggregations = []
+          @dimensions = HashWithIndifferentAccess.new
+        end
+
+        def dimension(*dimension_names,&block)
+          options = dimension_names.extract_options! || {}
+          dimension_names.flatten.each do |dim|
+            @dimensions[dim] = Dimension.new(dim,options,&block)
+          end
+        end
+
+        def measure(*measure_list)
+          options = measure_list.extract_options! || {}
+          measure_list.flatten.each { |m| self.measures[m] = options }
+        end
+
+        def aggregate_by(*aggregation_list)
+          self.aggregations << aggregation_list.flatten
+        end
+
+        def aggregate_all_combinations
+          dimension_names = dimensions.keys
+          (1..dimension_names.length).each do |combination_size|
+            dimension_names.combination(combination_size).each { |combo| aggregate_by *combo}
+          end
+        end
+        alias aggregate_by_all aggregate_all_combinations
+
+        def filter(&block)
+          return @filter unless block
+          @filter = block
+        end
+
+        def matches(message)
+          return true unless filter
+          filter.arity == 0 ? filter.call : filter.call(message)
+        end
+
+      end
+    end
+  end
+
+end

data/lib/wonkavision/plugins/analytics/aggregation/attribute.rb

@@ -0,0 +1,22 @@
+require "set"
+
+module Wonkavision
+  module Plugins
+    module Aggregation
+      class Attribute
+        attr_reader :name, :options
+
+        def initialize(name,options={})
+          @name = name
+          @options = options
+        end
+
+        def extract(message)
+          message[name.to_s]
+        end
+
+      end
+    end
+  end
+end
+

data/lib/wonkavision/plugins/analytics/aggregation/dimension.rb

@@ -0,0 +1,64 @@
+require "set"
+
+module Wonkavision
+  module Plugins
+    module Aggregation
+      class Dimension
+        attr_reader :name, :attributes, :options, :from
+        attr_writer :key, :sort, :caption
+
+        def initialize(name,options={},&block)
+          @name = name
+          @options = options
+          @attributes = HashWithIndifferentAccess.new
+          @from = options[:from]
+          key options[:key] if options[:key]
+          sort options[:sort] if options[:sort]
+          caption options[:caption] if options[:caption]
+          self.instance_eval(&block) if block
+          key name unless key
+        end
+
+        def attribute(*attribute_list)
+          raise "No attribute names were specified when calling '#attribute'" if
+            attribute_list.blank?
+
+          options = attribute_list.extract_options! || {}
+          attribute_list.flatten.each do |attribute|
+            @attributes[attribute] = Attribute.new(attribute,options)
+          end
+        end
+
+        def sort(sort_key = nil, options={})
+          return @sort || @key unless sort_key
+          attribute(sort_key, options) unless attributes[sort_key]
+          @sort = sort_key
+        end
+        alias :sort_by :sort
+
+        def caption(caption_key=nil, options={})
+          return @caption || @key unless caption_key
+          attribute(caption_key, options) unless attributes[caption_key]
+          @caption = caption_key
+        end
+
+        def key(key=nil, options={})
+          return @key unless key
+          attribute(key, options) unless attributes[key]
+          @key = key
+        end
+
+        def extract(data)
+          dimension_data = data[from.to_s] if from
+          dimension_data ||= data[name.to_s] if data[name.to_s].kind_of?(Hash)
+          dimension_data ||= data
+          attributes.values.inject({}) do |message,attribute|
+            message.tap { |m| m[attribute.name.to_s] = attribute.extract(dimension_data)};
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/wonkavision/plugins/analytics/aggregation/measure.rb

@@ -0,0 +1,240 @@
+# This class is based off of
+# https://github.com/josephruscio/aggregate and
+# https://github.com/afurmanov/aggregate
+#
+# Copyright (c) 2009 Joseph Ruscio
+#
+#Permission is hereby granted, free of charge, to any person
+#obtaining a copy of this software and associated documentation
+#files (the "Software"), to deal in the Software without
+#restriction, including without limitation the rights to use,
+#copy, modify, merge, publish, distribute, sublicense, and/or sell
+#copies of the Software, and to permit persons to whom the
+#Software is furnished to do so, subject to the following
+#conditions:
+#
+#The above copyright notice and this permission notice shall be
+#included in all copies or substantial portions of the Software.
+#
+#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#OTHER DEALINGS IN THE SOFTWARE.
+
+module Wonkavision
+  module Plugins
+    module Aggregation
+      class Measure
+        #The current average of all samples
+        attr_reader :mean
+
+        #The current number of samples
+        attr_reader :count
+
+        #The maximum sample value
+        attr_reader :max
+
+        #The minimum samples value
+        attr_reader :min
+
+        #The sum of all samples
+        attr_reader :sum
+
+        #The number of samples falling below the lowest valued histogram bucket
+        attr_reader :outliers_low
+
+        #The number of samples falling above the highest valued histogram bucket
+        attr_reader :outliers_high
+
+        DEFAULT_LOG_BUCKETS = 8
+
+        # The number of buckets in the binary logarithmic histogram (low => 2**0, high => 2**@@LOG_BUCKETS)
+        def log_buckets
+          @log_buckets
+        end
+
+        # Create a new Aggregate that maintains a binary logarithmic histogram
+        # by default. Specifying values for low, high, and width configures
+        # the aggregate to maintain a linear histogram with (high - low)/width buckets
+        def initialize (options={})
+          low = options[:low]
+          high = options[:high]
+          width = options[:width]
+          @log_buckets = options[:log_buckets] || DEFAULT_LOG_BUCKETS
+          @count = 0
+          @sum = 0.0
+          @sum2 = 0.0
+          @outliers_low = 0
+          @outliers_high = 0
+
+          # If the user asks we maintain a linear histogram where
+          # values in the range [low, high) are bucketed in multiples
+          # of width
+          if (nil != low && nil != high && nil != width)
+
+            #Validate linear specification
+            if high <= low
+              raise ArgumentError, "High bucket must be > Low bucket"
+            end
+
+            if high - low < width
+              raise ArgumentError, "Histogram width must be <= histogram range"
+            end
+
+            if 0 != (high - low).modulo(width)
+              raise ArgumentError, "Histogram range (high - low) must be a multiple of width"
+            end
+
+            @low = low
+            @high = high
+            @width = width
+          else
+            low ||= 1
+            @low = 1
+            @low = to_bucket(to_index(low))
+            @high = to_bucket(to_index(@low) + log_buckets - 1)
+          end
+
+          #Initialize all buckets to 0
+          @buckets = Array.new(bucket_count, 0)
+        end
+
+        # Include a sample in the aggregate
+        def add data
+
+          # Update min/max
+          if 0 == @count
+            @min = data
+            @max = data
+          else
+            @max = [data, @max].max
+            @min = [data, @min].min
+          end
+
+          # Update the running info
+          @count += 1
+          @sum += data
+          @sum2 += (data * data)
+
+          # Update the bucket
+          @buckets[to_index(data)] += 1 unless outlier?(data)
+        end
+        alias << add
+
+        def reject(data)
+          @min = Wonkavision::NaN
+          @max = Wonkavision::NaN
+          @count -= 1
+          @sum -= data
+          @sum2 -= (data * data)
+          @buckets[to_index(data)] -= 1 unless outlier?(data, true)
+        end
+        alias >> reject
+
+        def mean
+          @sum / @count
+        end
+
+        #Calculate the standard deviation
+        def std_dev
+          return Wonkavision::NaN unless @count > 1
+          Math.sqrt((@sum2.to_f - ((@sum.to_f * @sum.to_f)/@count.to_f)) / (@count.to_f - 1))
+        end
+
+        #Iterate through each bucket in the histogram regardless of
+        #its contents
+        def each
+          @buckets.each_with_index do |count, index|
+            yield(to_bucket(index), count)
+          end
+        end
+
+        #Iterate through only the buckets in the histogram that contain
+        #samples
+        def each_nonzero
+          @buckets.each_with_index do |count, index|
+            yield(to_bucket(index), count) if count != 0
+          end
+        end
+
+        # log2(x) returns j, | i = j-1 and 2**i <= data < 2**j
+        @@LOG2_DIVEDEND = Math.log(2)
+        def self.log2( x )
+          Math.log(x) / @@LOG2_DIVEDEND
+        end
+        private
+
+        def linear?
+          nil != @width
+        end
+
+        def outlier? (data, remove=false)
+          delta = remove ? -1 : 1
+          if data < @low
+            @outliers_low += delta
+          elsif data >= @high
+            @outliers_high += delta
+          else
+            return false
+          end
+        end
+
+        def bucket_count
+          if linear?
+            return (@high-@low)/@width
+          else
+            return log_buckets
+          end
+        end
+
+        def to_bucket(index)
+          if linear?
+            return @low + (index * @width)
+          else
+            return 2**(log2(@low) + index)
+          end
+        end
+
+        def right_bucket? index, data
+
+          # check invariant
+          raise unless linear?
+
+          bucket = to_bucket(index)
+
+          #It's the right bucket if data falls between bucket and next bucket
+          bucket <= data && data < bucket + @width
+        end
+
+        # A data point is added to the bucket[n] where the data point
+        # is less than the value represented by bucket[n], but greater
+        # than the value represented by bucket[n+1]
+
+        def to_index (data)
+
+          # basic case is simple
+          return log2([1,data/@low].max).to_i if !linear?
+
+          # Search for the right bucket in the linear case
+          @buckets.each_with_index do |count, idx|
+            return idx if right_bucket?(idx, data)
+          end
+          #find_bucket(0, bucket_count-1, data)
+
+          #Should not get here
+          raise "#{data}"
+        end
+
+        def log2(x)
+          self.class.log2(x)
+        end
+
+      end
+    end
+  end
+end
+