mongoid-report 0.1.9 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NWZhMzhiNGQzZDgxMTdlMjlhZGU2MTUyYTE5YzRjOGJmYTRhYTFjMw==
+    NzMwZDY2MzQxNmI5ZmJmYjUxZjhkZTJkM2VlOTg2OGI2YTBmNTBmNA==
   data.tar.gz: !binary |-
-    NWEzNzVjYzY1M2IwOTgxZmNmZTc1ZDY3N2VmNjY3NTMwZjFmZWMzYw==
+    ZTg4NzljZmFjNDRmOWNkZTExYWY0N2U3MGUxMTIwOTBkNzc0OTkxYw==
 SHA512:
   metadata.gz: !binary |-
-    Njc2NWE1OWIwYjE5NzJkNmJhOTYxYjM0MjNjMjMyMjkyMjMyZWM5ODVhZjc2
-    M2YzMDU3MmE3ZjI2MjFiNmVjMDE3MzA2ODIzMjIwOThlOThmOTViYjIxMmU4
-    MzY1Zjc1NjczYTk3NzViZDQyNGEzNjVhZTI2OTUzMzZlYjM1ZDU=
+    NzNiYzA4NmU2YTg1MjQyMDQxNzEwYmIwNzk1ZmRjZGYxOTNkODFjYTI0YWYy
+    MjU4MGU2YjQwZWZjOTUxZjdiMWZkMWE4OWRkYzIyZjFkMzEzMDdjYzZkMTJi
+    ZGRhNjFmOTc2ZWVjZTM5Y2MwMTJjNWJmYWUxMmYzMDNkNTdlODk=
   data.tar.gz: !binary |-
-    YjgyMjcyMWFmZjU2YzVmNGE3MzRjNTQ1ZTZhODQ4M2I3Y2RmOGQ1M2NiZjg4
-    YmY1Y2UzM2U3ZTI3MDlmODdkMmM3ODAyODVkNzVmY2EyNjA0NzZlNjM1MWVk
-    ODYwM2Y1OWI5YTlhOTM5ZmY1YmMxOWE2MWFjNGMwNzMzZTE2Y2Q=
+    NDMwMzRjMTg0YjE1NTE1ZTcwMmE1OTYxZTUxMTE3NmY1ZmMwNjM0MDAzNTk3
+    NjhlMzAxZjFiYTljMmI2MTQ2Y2Q5NWUyODIxNDYzOTczMGI1ZDgyNjAxNjFh
+    YmY3YTMzYWIwOTUyNmQ4ZjQyMTZlNjFmNDFiMDdjYzE0OGY3ZjM=

data/Gemfile.lock

@@ -1,19 +1,23 @@
 PATH
   remote: .
   specs:
-    mongoid-report (0.1.9)
+    mongoid-report (0.2.0)
       mongoid (> 3.0.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (3.2.18)
-      activesupport (= 3.2.18)
-      builder (~> 3.0.0)
-    activesupport (3.2.18)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
-    builder (3.0.4)
+    activemodel (4.1.4)
+      activesupport (= 4.1.4)
+      builder (~> 3.1)
+    activesupport (4.1.4)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    bson (2.3.0)
+    builder (3.2.2)
     celluloid (0.15.2)
       timers (~> 1.1.0)
     celluloid-io (0.15.0)
@@ -21,6 +25,7 @@ GEM
       nio4r (>= 0.5.0)
     coderay (1.1.0)
     columnize (0.3.6)
+    connection_pool (2.0.0)
     debugger (1.6.6)
       columnize (>= 0.3.1)
       debugger-linecache (~> 1.2.0)
@@ -42,7 +47,8 @@ GEM
     guard-rspec (4.2.8)
       guard (~> 2.1)
       rspec (>= 2.14, < 4.0)
-    i18n (0.6.9)
+    i18n (0.6.11)
+    json (1.8.1)
     listen (2.7.1)
       celluloid (>= 0.15.2)
       celluloid-io (>= 0.15.0)
@@ -50,15 +56,19 @@ GEM
       rb-inotify (>= 0.9)
     lumberjack (1.0.5)
     method_source (0.8.2)
-    mongoid (3.1.6)
-      activemodel (~> 3.2)
-      moped (~> 1.4)
-      origin (~> 1.0)
-      tzinfo (~> 0.3.29)
-    moped (1.5.2)
-    multi_json (1.10.1)
+    minitest (5.4.0)
+    mongoid (4.0.0)
+      activemodel (~> 4.0)
+      moped (~> 2.0.0)
+      origin (~> 2.1)
+      tzinfo (>= 0.3.37)
+    moped (2.0.0)
+      bson (~> 2.2)
+      connection_pool (~> 2.0)
+      optionable (~> 0.2.0)
     nio4r (1.0.0)
-    origin (1.1.0)
+    optionable (0.2.0)
+    origin (2.1.1)
     pry (0.9.12.6)
       coderay (~> 1.0)
       method_source (~> 0.8)
@@ -84,8 +94,10 @@ GEM
     rspec-support (3.0.0)
     slop (3.5.0)
     thor (0.19.1)
+    thread_safe (0.3.4)
     timers (1.1.0)
-    tzinfo (0.3.39)
+    tzinfo (1.2.1)
+      thread_safe (~> 0.1)
 
 PLATFORMS
   ruby

data/README.md

@@ -14,6 +14,7 @@ framework.
     include Mongoid::Document
 
     field :field1, type: Integer, default: 0
+    field :field2, type: Integer, default: 0
 
     field :day,    type: Date
   end
@@ -21,42 +22,11 @@ framework.
   class Report1
     include Mongoid::Report
 
-    column :field1, for: Model
-  end
-
-  class Report2
-    include Mongoid::Report
-
-    attach_to Model do
-      column :field1
-    end
-  end
-
-  class Report3
-    include Mongoid::Report
-
-    group_by :day, for: Model
-
-    column :field1, for: Model
-  end
-
-  class Report4
-    include Mongoid::Report
-
-    attach_to Model do
-      group_by :day
-
-      column :field1
-    end
-  end
-
-  class Report5
-    include Mongoid::Report
-
-    attach_to Model, as: 'summary-report' do
-      group_by :day
-
-      column :field1
+    report 'example' do
+      attach_to Model do
+        group_by :day
+        column :field1, collection: Model
+      end
     end
   end
 ```

data/lib/mongoid/report/attach_proxy.rb

@@ -2,12 +2,21 @@ module Mongoid
   module Report
 
     AttachProxy = Struct.new(:context, :collection, :options) do
-      attr_reader :attach_name
+      attr_reader :report_name
 
       def initialize(context, collection, options)
         # Lets remove as option because of passing to the next blocks options
-        @attach_name = options.delete(:as) || collection
-        options = options.merge(attach_name: attach_name, for: collection)
+        report_name = options.delete(:as)
+
+        if collection
+          report_name ||= Collections.name(collection)
+        end
+
+        options.merge!(
+          report_name:  report_name,
+          collection:   collection,
+        )
+
         super(context, collection, options)
       end
 
@@ -35,10 +44,22 @@ module Mongoid
         context.group_by(*fields, proxy_options)
       end
 
-      def filter(*fields)
+      def match(*fields)
+        proxy_options = fields.extract_options!
+        proxy_options.merge!(options)
+        context.match(*fields, proxy_options)
+      end
+
+      def query(*fields)
+        proxy_options = fields.extract_options!
+        proxy_options.merge!(options)
+        context.query(*fields, proxy_options)
+      end
+
+      def batches(*fields)
         proxy_options = fields.extract_options!
         proxy_options.merge!(options)
-        context.filter(*fields, proxy_options)
+        context.batches(*fields, proxy_options)
       end
     end
 

data/lib/mongoid/report/batches.rb

@@ -0,0 +1,41 @@
+module Mongoid
+  module Report
+
+    # Split the queries into threads.
+    Batches = Struct.new(:settings, :conditions) do
+      DEFAULT_THREAD_POOL_SIZE = 5
+
+      def initialize(settings = {}, conditions = {})
+        if settings.nil? || settings.empty?
+          settings = { 'pool_size' => DEFAULT_THREAD_POOL_SIZE }
+        end
+
+        super(settings, conditions)
+      end
+
+      def field
+        conditions.keys[0]
+      end
+
+      def range
+        conditions.values[0]
+      end
+
+      def map
+        range.each_slice(size.ceil).map do |r|
+          yield r
+        end
+      end
+
+      def size
+        range.count.to_f / settings['pool_size'].to_f
+      end
+
+      def present?
+        settings['pool_size'].present? &&
+          conditions.present?
+      end
+    end
+
+  end
+end

data/lib/mongoid/report/collection.rb

@@ -3,7 +3,7 @@ require 'delegate'
 module Mongoid
   module Report
 
-    class Collection < SimpleDelegator
+    class Collection
       def initialize(context, rows, fields, columns, mapping)
         @context = context
         @rows    = rows
@@ -15,7 +15,7 @@ module Mongoid
 
       class Rows < SimpleDelegator ; end
 
-      attr_reader :rows
+      attr_reader :context, :rows
 
       def headers
         @fields
@@ -29,13 +29,14 @@ module Mongoid
             # all summaried mongo columns and then apply dynamic columns
             # calculations.
             next if @columns.has_key?(field)
+            next unless row[field].is_a?(Fixnum) || row[field].is_a?(Float)
             summary[field] += row[field]
           end
 
           # Apply dynamic columns for summarized row
           @columns.each do |name, function|
             next unless @fields.include?(name)
-            summary[name] = function.call(@context, row, { mapping: @mapping, summary: true })
+            summary[name] = function.call(@context, summary, { mapping: @mapping, summary: true })
           end
 
           summary

data/lib/mongoid/report/collections.rb

@@ -0,0 +1,16 @@
+module Mongoid
+  module Report
+
+    class Collections
+      def self.get(collection_name)
+        Mongoid.session(:default)[collection_name]
+      end
+
+      def self.name(model)
+        model && model.respond_to?(:collection) ?
+          model.collection.name : model
+      end
+    end
+
+  end
+end

data/lib/mongoid/report/input.rb

@@ -0,0 +1,17 @@
+module Mongoid
+  module Report
+
+    class Input
+      attr_accessor :collection_name
+
+      def present?
+        collection_name.present?
+      end
+
+      def collection
+        @collection ||= Mongoid::Report::Collections.get(collection_name)
+      end
+    end
+
+  end
+end

data/lib/mongoid/report/merger.rb

@@ -0,0 +1,29 @@
+module Mongoid
+  module Report
+
+    # We are using this class to combine results by group by fields.
+    Merger = Struct.new(:groups) do
+      def do(rows)
+        # Merge by groups.
+        rows
+          .group_by { |row| groups.map { |group| row[group] }.join('-') }
+          .values
+          .map { |array_row| combine(array_row) }
+      end
+
+      private
+
+      def combine(rows)
+        rows.inject(Hash.new {|h,k| h[k] = 0}) do |row, lines|
+          lines.each do |key, value|
+            next row[key] = value if groups.include?(key)
+            row[key] += value
+          end
+
+          row
+        end
+      end
+    end
+
+  end
+end

data/lib/mongoid/report/output.rb

@@ -0,0 +1,32 @@
+module Mongoid
+  module Report
+
+    class Output
+      attr_accessor :collection_name, :options
+
+      def do(rows)
+        drop()
+        collection.insert(rows)
+      end
+
+      def present?
+        collection_name.present?
+      end
+
+      def drop
+        return collection.drop() unless options[:drop].present?
+
+        # We will use custom way for dropping the collection or removing the
+        # records partially
+        collection.find(options[:drop]).remove_all()
+      end
+
+      private
+
+      def collection
+        @collection ||= Collections.get(collection_name)
+      end
+    end
+
+  end
+end

data/lib/mongoid/report/queries_builder.rb

@@ -1,7 +1,7 @@
 module Mongoid
   module Report
 
-    QueriesBuilder = Struct.new(:settings) do
+    QueriesBuilder = Struct.new(:configuration) do
       def do
         [].tap do |queries|
           queries.concat([{ '$project' => project_query }])
@@ -13,26 +13,22 @@ module Mongoid
       private
 
       def groups
-        @group_by ||= settings.fetch(:group_by, [])
+        configuration[:group_by]
       end
 
       def fields
-        @fields ||= settings[:fields].select do |field, _|
-          !settings[:columns].include?(field.to_sym)
-        end
-      end
-
-      def in_fields
-        @in_fields ||= fields.keys
-      end
+        @fields ||= begin
+          columns = configuration[:columns].keys
 
-      def output_fields
-        @output_fields ||= fields.values
+          configuration[:fields].select do |field|
+            !columns.include?(field.to_sym)
+          end
+        end
       end
 
       def all_fields
         [:_id]
-          .concat(in_fields)
+          .concat(fields)
           .concat(groups)
       end
 
@@ -52,7 +48,8 @@ module Mongoid
             hash.merge!(group => GROUP_TEMPLATE % group)
           end
 
-          in_fields.inject(query) do |hash, field|
+          fields.inject(query) do |hash, field|
+            next hash if groups.include?(field)
             hash.merge!(field => { '$sum' => GROUP_TEMPLATE % field })
           end
         end
@@ -71,8 +68,9 @@ module Mongoid
             end
           end
 
-          fields.inject(query) do |hash, (field, name)|
-            hash.merge!(name => "$#{field}")
+          fields.inject(query) do |hash, field|
+            next hash if groups.include?(field)
+            hash.merge!(field => "$#{field}")
           end
         end
       end

data/lib/mongoid/report/report_proxy.rb

@@ -1,13 +1,25 @@
+require 'securerandom'
+
 module Mongoid
   module Report
 
     ReportProxy = Struct.new(:context, :name) do
-      def attach_to(model, options = {}, &block)
-        as = options.fetch(:as) { model.collection.name }
+      def attach_proxy
+        @attach_proxy ||= begin
+          AttachProxy.new(context, nil, report_module: name)
+        end
+      end
+      delegate :column, :columns, :mapping, :group_by, :match,
+        :batches, :query, to: :attach_proxy
+
+      def attach_to(*fields, &block)
+        options = fields.extract_options!
+        model   = fields[0]
 
-        options.merge!(as: "#{name}-#{as}")
+        report_name = options.delete(:as) || Collections.name(model)
+        options.merge!(report_module: name, as: report_name)
 
-        context.attach_to(model, options, &block)
+        context.attach_to(*fields, options, &block)
       end
     end
 

data/lib/mongoid/report/scope.rb

@@ -1,7 +1,9 @@
+require 'set'
+
 module Mongoid
   module Report
 
-    Scope = Struct.new(:context, :report_name) do
+    Scope = Struct.new(:context, :report_module, :report_name) do
       def query(conditions = {})
         queries.concat([conditions]) unless conditions.empty?
         self
@@ -11,41 +13,112 @@ module Mongoid
       def yield
         return self if @yielded
 
-        queries.concat(context.queries(report_name))
+        queries.concat(context.queries(report_module, report_name))
         @yielded = true
 
         self
       end
 
+      def out(collection_name, options = {})
+        output.collection_name = collection_name
+        output.options = options
+        self
+      end
+
+      def in(collection_name)
+        input.collection_name = collection_name
+        self
+      end
+
+      def all_in_batches(aggregation_queries)
+        # Lets assume we have only one field for making splits for the
+        # aggregation queries.
+        rows = []
+
+        threads = batches.map do |r|
+          # For now we are supporting only data fields for splitting up the
+          # queries.
+          range_match = r.map { |time| time.to_date.mongoize }
+
+          Thread.new do
+            q =
+              ['$match' => { batches.field => { '$gte' => range_match.first, '$lte' => range_match.last } }] +
+              aggregation_queries
+
+            # if groups == [batch.field]
+            rows.concat(Array(collection.aggregate(q)))
+          end
+        end
+        threads.map(&:join)
+
+        merger = Mongoid::Report::Merger.new(groups)
+        merger.do(rows)
+      end
+
+      def all_inline(aggregation_queries)
+        Array(collection.aggregate(aggregation_queries))
+      end
+
       def all
         self.yield unless yielded?
 
         aggregation_queries = compile_queries
-        rows = klass.collection.aggregate(aggregation_queries)
+
+        rows = if batches.present?
+          all_in_batches(aggregation_queries)
+        else
+          all_inline(aggregation_queries)
+        end
+
+        # in case if we want to store rows to collection
+        if output.present?
+          output.do(rows)
+        end
 
         Collection.new(context, rows, fields, columns, mapping)
       end
 
+      def in_batches(conditions)
+        batches.conditions = conditions
+        self
+      end
+
       private
 
       def compile_queries
-        compiled = queries.map do |query|
-          next query unless query.has_key?("$match")
+        compiled = Set.new
+
+        queries.each do |query|
+          next compiled << query if query.has_key?("$project") || query.has_key?('$group')
 
           query.deep_dup.tap do |new_query|
             new_query.each do |function_name, values|
-              values.each do |name, value|
-                if value.respond_to?(:call)
-                  value = value.call(context)
+
+              if values.respond_to?(:call)
+                new_query[function_name] = values.call(context)
+              else
+                values.each do |name, value|
+                  if value.respond_to?(:call)
+                    value = value.call(context)
+                  end
+
+                  unless value.present?
+                    # In case we don't have value for applying match, lets skip
+                    # this type of the queries.
+                    new_query.delete(function_name)
+                  else
+                    new_query[function_name][name] = value
+                  end
                 end
 
-                new_query[function_name][name] = value
-              end
-            end
+              end # values.is_a?(Proc)
+            end # new_query.each
+
+            compiled << new_query if new_query.present?
           end
         end
 
-        compiled
+        compiled.to_a
       end
 
       def yielded?
@@ -56,22 +129,55 @@ module Mongoid
         @queries ||= []
       end
 
-      def klass
-        context.report_module_settings[report_name][:for]
+      # Different usage for this method:
+      # - attach_to method contains collection name as first argument
+      # - attach_to method contains mongoid model
+      # - aggregate_for method contains attach_to proc option for calculating
+      # collection name.
+      def collection
+        @collection ||= begin
+          # In case if we are using dynamic collection name calculated by
+          # passing attach_to proc to the aggregate method.
+          if input.present?
+            # Using default session to mongodb we can automatically provide
+            # access to collection.
+            input.collection
+          else
+            klass = context.report_module_settings[report_module][:reports][report_name][:collection]
+            Collections.get(klass)
+          end
+        end
+      end
+
+      def batches
+        @batches ||= Mongoid::Report::Batches.new(
+          context.batches(report_module, report_name))
+      end
+
+      def output
+        @output ||= Mongoid::Report::Output.new
+      end
+
+      def input
+        @input ||= Mongoid::Report::Input.new
+      end
+
+      def groups
+        @groups ||= context.groups(report_module, report_name)
       end
 
       def fields
         # We need to use here only output field names it could be different
         # than defined colunms, Example: field1: 'report-field-name'
-        context.report_module_settings[report_name][:fields].values
+        context.fields(report_module, report_name)
       end
 
       def columns
-        context.report_module_settings[report_name][:columns]
+        context.columns(report_module, report_name)
       end
 
       def mapping
-        context.report_module_settings[report_name][:mapping]
+        context.mapping(report_module, report_name)
       end
     end