aggrobot 0.0.2 → 0.1.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjhjNWE3NWU3M2IxOWQxZWIxMjIwYTVjMmQ5Mjc5ZDFmYTRhZTI5Yg==
+    MDBhY2U5ZGY4YWYzZTNmMjE5MWFlNmQ4OGYyYWY5Y2M4YWE3MWUwMw==
   data.tar.gz: !binary |-
-    OTc4YjYzNjlmMjdlMjIzNzY4MTcwZTA5YWUwNjk2MTkyMzM4M2Q5MQ==
+    MjZhMzBmMTFhZTg1MmE5ZWZhMGNlMjI5MGQwZTY2NWYxMTk0MWM0MA==
 SHA512:
   metadata.gz: !binary |-
-    ODk4NjY1NGJjZTIxMWZjMWZhODk4OTI4YTBhYmFiMzRjMDM1NjkzOTM4YTJi
-    OTdkYTA1MjEwNTBhODk2OGQwODU0NDIyNzA5YzI5OWQ3OTJlN2I4MWI4ZjMw
-    NzczYjY1OWRlYzY4ZDI5ZjE1MDcwMmZkZTdkYTYyMzAwYzgyNTE=
+    ZjhhMzcyMmU0ZjU4MThhNmY4NGI5MDI5MmQ4YWNkMzMwMDg1M2Y4ZDlhM2Yw
+    MzNjNWNkY2RlNWUzNDc2YTViNTVlM2I1YjUxYjJkN2M2OTM3ODEwMjJmNjQ0
+    NjE3YTc1YTZkM2FmMjZlYjQ4YmFkYjljY2IxYWYxNzAzNGNlNjE=
   data.tar.gz: !binary |-
-    YTA2MjhlMzI0ZjU1YzY4NWRlYWE3Y2VhMjRiOGJlNzA5YjljMWJhYWM1MGY4
-    MjZmZTA0MjEwOGNkNmZiMDc4ZWJmMmIwOWNjNTZkMjQ5MTgxYTFkMWRmZTcx
-    NjZjNDA1YzcxODgzYTRkZmY2N2VkZjU4OGJlMDI3YTU3YWI4ZTM=
+    ZjY4Y2IwYzBkZmJkNDg3MjRmMzBkNGM1ZGNjODkxNjgyNDdhYTdkMmI5YTEz
+    NDI4N2ZhZGUzMjZjMWJmNmQ1MWRhYTkwOTA5OWM3ZGI2ZWVlYTI0NjI4Njdh
+    YmMyZTMxMTBhNGI4MDg0YzZhOTllNmRiY2MwMmVkYjM2MDVhMWI=

data/Gemfile

@@ -1,8 +1,11 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in aggrobot.gemspec
+gem 'activerecord'
+
 gemspec
 
+
 group :test do
   gem 'guard-rspec', require: false
   gem 'growl', require: false

data/README.md

@@ -1,6 +1,15 @@
 # Aggrobot
 
-TODO: Write a gem description
+Aggrobot is an aggregation framework in Ruby. It provides a powerful DSL to perform aggregations over large dataset. It has been tested to work with MySQL, Postgres and SQLite. 
+
+Many other features are provided:
+* Bucketing over data ranges. For e.g. grouping orders over price ranges like 100-200, 200-300 and 300+
+* Grouping over top **n groups**. For e.g. group top 2 selling products based on quantity, while group rest of the products in a single bucket of **others**
+*  Sub-aggregations (avg, sum, add, multiply, divide, percent, etc) on columns are also provided.
+
+All the aggregations are calculated in the database and only the aggregated data is sent over to Ruby, to keep it performant. This greatly speeds up performance and reduces the memory requirement, had the aggregations been done directly in Ruby.
+
+Aggrobot also allows nested aggregations and each level of aggregation can be passed around as a code block and used in higher level aggregations. This provides great amount of code-reuse.
 
 ## Installation
 
@@ -19,6 +28,7 @@ Or install it yourself as:
 ## Usage
 
 TODO: Write usage instructions here
+TODO: Write example usage here
 
 ## Contributing
 

data/aggrobot.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ['shadab.ansari@gmail.com']
   spec.description   = %q{Easy and performant aggregation for rails}
   spec.summary       = %q{Rails aggregation library}
-  spec.homepage      = ''
+  spec.homepage      = 'https://github.com/shadabahmed/aggrobot'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files`.split($/)
@@ -18,9 +18,17 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'rails', '~> 4.0'
+  spec.add_dependency 'rails', '>=4.0'
+  spec.add_dependency 'activerecord', '>=4.0'
+  spec.add_dependency 'arel'
+
+  spec.required_ruby_version = '>= 1.9.3'
+
   spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency('rspec', ['~> 2.14.1'])
   spec.add_development_dependency('rdoc')
+  spec.add_development_dependency('factory_girl')
+  spec.add_development_dependency('database_cleaner')
 end

data/lib/aggrobot.rb

@@ -1,3 +1,4 @@
+require 'active_record'
 require 'aggrobot/railtie'
 require 'active_support/core_ext/module/delegation.rb'
 require 'active_support/core_ext/hash/indifferent_access'
@@ -9,11 +10,13 @@ require 'aggrobot/query_planner'
 require 'aggrobot/aggregator'
 require 'aggrobot/aggrobot'
 
-
 module Aggrobot
 
-  DEFAULT_GROUP_BY = SqlFunctions.sanitize('aggrobot_default_group')
+  DEFAULT_GROUP_BY = 1 #'aggrobot_default_group'
 
+  # when
+  #     collection is given, starts aggregation by evaluating block on collection
+  #     collection is not given, starts aggregation by evaluating block on full data set
   def self.start(collection = nil, block_arg = nil, block_opts = nil, &block)
     block_opts ||= block_arg if block
     block = block_arg if block_arg && block_arg.respond_to?(:call)
@@ -29,8 +32,9 @@ module Aggrobot
     block
   end
 
+  # sets ROUNDING_DIGITS to percent_precision, default is 2
   def self.setup(app)
-    SqlFunctions.const_set(:ROUNDING_DIGITS, app.config.aggrobot.percent_precision || 2)
+    SQLFunctions.setup(app.config.aggrobot.percent_precision)
   end
 
 end

data/lib/aggrobot/aggregator.rb

@@ -11,7 +11,7 @@ module Aggrobot
       self.collection(collection) if collection
     end
 
-
+    # returns hash of group label(s) as key and actual column(s) as value 
     def group_labels(map = nil, &block)
       if map || block
         if map.is_a?(Hash)
@@ -24,16 +24,25 @@ module Aggrobot
       end
     end
 
+    # returns collection if it is ActiveRecord::Relation or ActiveRecord::Base
+    # raises error when collection is none of the above
+    # returns @collection otherwise (which is nil)
     def collection(values = nil)
       if values
-        raise_error 'Collection should be an ActiveRecord::Relation or ActiveRecord::Base' unless
-            [ActiveRecord::Relation, ActiveRecord::Base].any?{|m| values.is_a?(m) }
+        if !values.is_a?(ActiveRecord::Relation) && values < ActiveRecord::Base
+          values = values.unscoped
+        end
+        raise_error 'Collection should be an ActiveRecord::Relation or ActiveRecord::Base' unless values.is_a?(ActiveRecord::Relation)
         @collection = values
       else
         @collection
       end
     end
 
+    # when 
+    #     : opts is nil, groups by group on @collection
+    #     : opts is a map as {limit_to: limit}, creats groups by group on @collection with a limit
+    #     : opts is a map as {buckets: [list_items]}, creats groups by [list_items] on @collection
     def group_by(group, opts = nil)
       raise_error "Group_by takes only symbol or a string as argument" unless group.is_a?(Symbol) or group.is_a?(String)
       @query_planner = QueryPlanner.create(@collection, group, opts)
@@ -50,7 +59,11 @@ module Aggrobot
       end
     end
 
-    def set(name = nil, opts)
+    # creates attribute map
+    # when: 
+    #     given as hash, sets all keys as attributes to show and values as columns to fetch
+    #     given as list (of 2 items), first item is key to show and second item is column to fetch
+    def select(name = nil, opts)
       if opts.is_a? Hash
         @attribute_mapping.merge!(opts)
       elsif name && opts
@@ -74,7 +87,7 @@ module Aggrobot
     private
 
     def extra_columns
-      @attribute_mapping.values
+      @attribute_mapping.map{|k, v| "#{v} as #{k}"}
     end
 
     def extra_attributes

data/lib/aggrobot/aggrobot.rb

@@ -1,13 +1,13 @@
 module Aggrobot
   class Aggrobot
-
-    include SqlFunctions
     include Helper
 
-    delegate :collection, :group_by, :default_groups, :override, :set, :group_labels, :to => :@aggregator
+    delegate :collection, :group_by, :default_groups, :override, :select, :group_labels, :to => :@aggregator
+    delegate :count,:sanitize, :desc, :asc, :count, :unique_count, :max, :min, :sum, :avg, :average,
+             :group_collect, :percent, :multiply, :divide, :to => SQLFunctions
 
-    def run(block)
-      instance_eval(&block)
+    def run(block, args = {})
+      instance_exec(args, &block)
     end
 
     def initialize(caller_context, collection = nil)
@@ -19,18 +19,21 @@ module Aggrobot
       @caller_context.send method, *args, &block
     end
 
+    # creates top level data structure as hash and call block to process further
     def hash(collection = nil, opts = {}, &block)
       self.collection(collection) if collection
       @top_level_object = ActiveSupport::HashWithIndifferentAccess.new
       proceed(block, opts)
     end
 
+    # creates top level data structure as array and call block to process further
     def list(collection = nil, opts = {}, &block)
       self.collection(collection) if collection
       @top_level_object = []
       proceed(block, opts)
     end
 
+    # sets default/current values to top_level_object hash/list
     def default(default_val = nil, &block)
       block = block_from_args(default_val, block, false)
       default_val = ::Aggrobot.start(collection, &block) if block
@@ -39,6 +42,7 @@ module Aggrobot
 
     alias set_current_value default
 
+    # sets default group attrs as a hash, if opts is passed as param
     def default_group_attrs(opts = nil)
       if opts
         raise_error 'Arguments must be  a hash' unless opts.is_a?(Hash)
@@ -48,10 +52,15 @@ module Aggrobot
       end
     end
 
+    alias default_values default_group_attrs
+
+    # returns top level object hash/list
     def current_value
       @top_level_object
     end
 
+    # starts aggrobot on collection and block, when block is given and
+    # adds {attribute: value} pair to the top level object
     def attr(attribute, value = nil, &block)
       block = block_from_args(value, block, false)
       raise_error 'attr can only be used with a hash type' unless @top_level_object.is_a?(Hash)
@@ -61,27 +70,29 @@ module Aggrobot
       @top_level_object[attribute] = value
     end
 
+    # gets attribute's value from top level object, only works when top level is hash
     def get_attr(attribute)
       @top_level_object[attribute]
     end
 
-    def collect_each_group_attributes
-      each_group do |attr|
-        attr
-      end
-    end
-
     def each_group(block_arg = nil, &block)
-      block = block_from_args(block_arg, block)
+      block = block_from_args(block_arg, block, false)
       @aggregator.yield_results do |attrs, group_name, sub_collection|
         attrs = @default_group_attrs.merge(attrs) if @default_group_attrs
         block_value = ::Aggrobot.start(sub_collection) do
-          instance_exec(attrs, &block)
+          if block
+            instance_exec(attrs, &block)
+          else
+            attrs
+          end
         end
         update_top_level_obj(group_name, block_value)
       end
     end
 
+    alias iterate each_group
+    alias recurse each_group
+
     def evaluate(block_arg = nil, &block)
       block = block_from_args(block_arg, block)
       list(&block).first

data/lib/aggrobot/errors.rb

@@ -0,0 +1,6 @@
+module Aggrobot
+  class Error < StandardError
+  end
+  class ArgumentError < ::ArgumentError
+  end
+end

data/lib/aggrobot/query_planner.rb

@@ -1,36 +1,25 @@
+require 'aggrobot/query_planner/parameters_validator'
 require 'aggrobot/query_planner/default_query_planner'
 require 'aggrobot/query_planner/group_limit_query_planner'
 require 'aggrobot/query_planner/bucketed_groups_query_planner'
 
+
+# plans queries in a Aggrobot
 module Aggrobot::QueryPlanner
 
+  # creates query object
   def self.create(collection, group_by, opts = nil)
     case
     when opts.nil?
       DefaultQueryPlanner.new(collection, group_by)
     when opts.key?(:limit_to)
+      # GROUP attrs by 'group_by' with limit
       GroupLimitQueryPlanner.new(collection, group_by, opts)
     when opts.key?(:buckets)
+      # GROUP attrs by 'group_by' in buckets of opts[:buckets], e.g. 1..100, 101..200 etc
       BucketedGroupsQueryPlanner.new(collection, group_by, opts)
+    else
+      raise ArgumentError.new "Invalid options to group_by : #{opts}"
     end
   end
-
-  module ParametersValidator
-    def self.validate_options(opts, required_parameters, optional_parameters)
-      params = opts.keys
-      # raise errors for required parameters
-      raise_argument_error(opts, required_parameters, optional_parameters) unless (required_parameters - params).empty?
-      # raise errors if any extra arguments given
-      raise_argument_error(opts, required_parameters, optional_parameters) unless (params - required_parameters - optional_parameters).empty?
-    end
-
-    def self.raise_argument_error(opts, required_parameters, optional_parameters)
-      raise ArgumentError, <<-ERR
-          Wrong arguments given - #{opts}
-          Required parameters are #{required_parameters}
-          Optional parameters are #{optional_parameters}
-      ERR
-    end
-  end
-
 end

data/lib/aggrobot/query_planner/agg.rb

@@ -0,0 +1,28 @@
+
+route = Route.first
+Infinity = (1.0/0)
+
+def los_labels
+    ->(label){
+      range = label.match(/(?<begin>\-?\d+)?(?<separator>[\-\<])?(?<end>\-?\d+)?/)
+      case range[:separator]
+      when '<'
+        "#{range[:end]}+"
+      when '-'
+        "#{range[:begin]} to #{range[:end]}"
+      else
+        label
+      end
+    }
+  end
+
+los_agg = Aggrobot.start(route.passengers) do |attrs|
+  hash do
+    group_by :length_of_stay, limit_to: 4, sort_by: count, other_group: 'others'
+    set advance: :ap, ptype: :pax_type, pct: percent(count, attrs[:count])
+    
+    each_group do |attrs| 
+      attrs
+    end
+  end
+end

data/lib/aggrobot/query_planner/bucketed_groups_query_planner.rb

@@ -3,19 +3,21 @@ module Aggrobot
     class BucketedGroupsQueryPlanner < DefaultQueryPlanner
 
       def initialize(collection, group, opts = {})
-        ParametersValidator.validate_options(opts, [:buckets], [:keep_empty])
+        required_params = [:buckets]
+        optional_params = [:keep_empty]
+        validate_options(opts, required_params, optional_params)
         raise_error 'Need to set group first' unless group
         super(collection, group)
         create_query_map(opts[:buckets])
         @keep_empty = opts[:keep_empty]
       end
 
-      def sub_query(group_name)
-        @query_map[group_name]
+      def sub_query(group_value)
+        @query_map[group_value]
       end
 
       def query_results(extra_cols = [])
-        return empty_default_groups if collection_is_none?
+        return empty_buckets if collection_is_none?
         results = collect_query_results(extra_cols)
         results.reject! { |r| r[1] == 0 } unless @keep_empty
         results
@@ -24,24 +26,25 @@ module Aggrobot
       private
 
       def collect_query_results(extra_cols)
-        columns = ['', SqlFunctions.count] + extra_cols
-        @query_map.collect do |group_name, query|
-          sanitized_group_name = SqlFunctions.sanitize(group_name)
-          columns[0] = sanitized_group_name
-          results = query.group(sanitized_group_name).limit(1).pluck(*columns).first
-          @query_map[group_name] = @query_map[group_name].none unless results
-          results || [group_name, 0]
+        columns = [SQLFunctions.count] + extra_cols
+        @query_map.collect do |group_value, query|
+          results = query.limit(1).pluck(*columns).flatten
+          if results[0] == 0
+            @query_map[group_value] = @query_map[group_value].none
+            results = [0]
+          end
+          results.unshift(group_value)
         end
       end
 
-      def empty_default_groups
+      def empty_buckets
         @keep_empty ? @query_map.keys.collect { |k| [k, 0] } : []
       end
 
-      def create_query_map(groups)
+      def create_query_map(buckets)
         @query_map = {}
-        groups.each do |group|
-          @query_map[group.to_s] = @collection.where(@group => group)
+        buckets.each do |bucket|
+          @query_map[bucket] = @collection.where(group_condition(bucket))
         end
       end
 

data/lib/aggrobot/query_planner/default_query_planner.rb

@@ -1,23 +1,43 @@
 module Aggrobot
   module QueryPlanner
     class DefaultQueryPlanner
-      include Aggrobot::Helper
+      include ParametersValidator
 
-      def initialize(collection, group)
-        @collection, @group = collection, group
+      def initialize(collection, group = DEFAULT_GROUP_BY)
+        @collection, @group = validate_and_extract_relation(collection), group
       end
 
-      def sub_query(group_name)
-        @group == DEFAULT_GROUP_BY ? @collection : @collection.where(@group => group_name)
+      def sub_query(group_value)
+        if @group == DEFAULT_GROUP_BY
+          @collection
+        else
+          @collection.where(group_condition(group_value))
+        end
       end
 
       def query_results(extra_cols = [])
         return [] if collection_is_none?
-        columns = [@group, SqlFunctions.count] + extra_cols
-        results_query.pluck(*columns)
+        if @group.is_a? Array
+          columns = @group + [SQLFunctions.count] + extra_cols
+          results_query.pluck(*columns).collect do |result_row|
+            [result_row[0..(@group.count - 1)]] + result_row[@group.count..-1]
+          end
+        else
+          columns = [@group, SQLFunctions.count] + extra_cols
+          results_query.pluck(*columns)
+        end
       end
 
       protected
+
+      def group_condition(group_value)
+        if @group.is_a?(Array)
+          Hash[@group.zip(group_value)]
+        else
+          {@group => group_value}
+        end
+      end
+
       def results_query
         @result_query ||= @collection.group(@group)
       end
@@ -25,7 +45,6 @@ module Aggrobot
       def collection_is_none?
         @collection.extending_values.include?(ActiveRecord::NullRelation)
       end
-
     end
   end
 end