mochigome 0.0.3 → 0.0.4

data/lib/query.rb

@@ -1,199 +1,346 @@
 require 'rgl/adjacency'
+require 'rgl/traversal'
 
 module Mochigome
   class Query
-    def initialize(layer_types, name = "report")
+    def initialize(layer_types, options = {})
       # TODO: Validate layer types: not empty, AR, act_as_mochigome_focus, graph correctly, no repeats
       @layer_types = layer_types
-      @name = name
-    end
-
-    def run(objs)
-      objs = [objs] unless objs.is_a?(Enumerable)
-      return DataNode.new(:report, @name) if objs.size == 0 # Empty DataNode for empty input
+      @layers_path = self.class.path_thru(layer_types)
 
-      unless objs.all?{|obj| obj.class == objs.first.class}
-        raise QueryError.new("Query target objects must all be the same type")
+      @name = options.delete(:root_name).try(:to_s) || "report"
+      @access_filter = options.delete(:access_filter) || lambda {|cls| {}}
+      aggregate_sources = options.delete(:aggregate_sources) || []
+      unless options.empty?
+        raise QueryError.new("Unknown options: #{options.keys.inspect}")
       end
 
-      unless @layer_types.any?{|layer| objs.first.is_a?(layer)}
-        raise QueryError.new("Query target object type must be in the query layer list")
+      @ids_rel = self.class.relation_over_path(@layers_path).
+        project(@layers_path.map{|m|
+          Arel::Table.new(m.table_name)[m.primary_key]
+        })
+      @ids_rel = access_filtered_relation(@ids_rel, @layers_path)
+
+      # TODO: Validate that aggregate_sources is in the correct format
+      aggs_by_model = {}
+      aggregate_sources.each do |focus_cls, data_cls|
+        aggs_by_model[focus_cls] ||= []
+        aggs_by_model[focus_cls] << data_cls
       end
 
-      # Used to provide debugging information in the root DataNode comment
-      assoc_path = ["== #{objs.first.class.name} =="]
+      @aggregate_rels = {}
+      aggs_by_model.each do |focus_model, data_models|
+        # Need to do a relation over the entire path in case query runs
+        # on something other than the focus model layer.
+        # TODO: Would be better to only do this if necesssitated by the
+        # conditions supplied to the query when it is ran, and/or
+        # the access filter.
+        focus_rel = self.class.relation_over_path(@layers_path)
 
-      # Start at the layer for objs, and descend downwards through layers after that
-      #TODO: It would be really fantastic if I could just use AR eager loading for this
-      downwards_layers = @layer_types.drop_while{|cls| !objs.first.is_a?(cls)}
-      root = DataNode.new(:report, @name)
-      root << objs.map{|obj| DataNode.new(
-        obj.mochigome_focus.type_name,
-        obj.mochigome_focus.name,
-        [{:obj => obj}]
-      )}
-      cur_layer = root.children
-      downwards_layers.drop(1).each do |cls|
-        new_layer = []
-        assoc = Query.edge_assoc(cur_layer.first[:obj].class, cls)
-
-        assoc_str = "-> #{cls.name} via #{cur_layer.first[:obj].class.name}##{assoc.name}"
-        if assoc.through_reflection
-          assoc_str << " (thru #{assoc.through_reflection.name})"
-        end
-        assoc_path.push assoc_str
-
-        cur_layer.each do |datanode|
-          # FIXME: Don't assume that downwards means plural association
-          # TODO: Are there other ways context could matter besides :through assocs?
-          # i.e. If C belongs_to A and also belongs_to B, and layer_types = [A,B,C]
-          # TODO: What if a through reflection goes through _another_ through reflection?
-          if assoc.through_reflection
-            datanode[:obj].send(assoc.through_reflection.name).each do |through_obj|
-              # TODO: Don't assume that through means singular!
-              obj = through_obj.send(assoc.source_reflection.name)
-              subnode = datanode << DataNode.new(
-                obj.mochigome_focus.type_name,
-                obj.mochigome_focus.name,
-                {:obj => obj, :through_obj => through_obj}
-              )
-              new_layer << subnode
-            end
-          else
-            #FIXME: Not DRY
-            datanode[:obj].send(assoc.name).each do |obj|
-              subnode = datanode << DataNode.new(
-                obj.mochigome_focus.type_name,
-                obj.mochigome_focus.name,
-                [{:obj => obj}]
-              )
-              new_layer << subnode
+        # TODO: Properly handle focus model that is not in layer types list
+        key_path = @layers_path.take_while{|m| m != focus_model} + [focus_model]
+        key_cols = key_path.map{|m|
+          Arel::Table.new(m.table_name)[m.primary_key]
+        }
+
+        @aggregate_rels[focus_model] = {}
+        data_models.each do |data_model|
+          f2d_path = self.class.path_thru([focus_model, data_model]) #TODO: Handle nil here
+          agg_path = nil
+          f2d_path.reverse.each do |link_model|
+            if @layers_path.include?(link_model)
+              agg_path = f2d_path.drop_while{|m| m != link_model}
+              break
             end
           end
+          # TODO: Properly handle focus model that is not in layer types list
+          fail unless agg_path
+
+          agg_data_rel = self.class.relation_over_path(agg_path, focus_rel.dup)
+          agg_data_rel = access_filtered_relation(agg_data_rel, @layers_path + agg_path)
+          data_tbl = Arel::Table.new(data_model.table_name)
+          agg_fields = data_model.mochigome_aggregation_settings.options[:fields].reject{|a| a[:in_ruby]}
+          agg_data_rel.project
+          agg_fields.each_with_index do |a, i|
+            agg_data_rel.project(a[:value_proc].call(data_tbl).as("d#{i}"))
+          end
+
+          @aggregate_rels[focus_model][data_model] = (0..key_cols.length).map{|n|
+            lambda {|cond|
+              d_rel = agg_data_rel.dup
+              d_cols = key_cols.take(n) + [Arel::Table.new(data_model.table_name)[data_model.primary_key]]
+              d_cols.each_with_index do |col, i|
+                d_rel.project(col.as("g#{i}")).group(col)
+              end
+              d_rel.where(cond)
+
+              a_rel = Arel::SelectManager.new(
+                Arel::Table.engine,
+                Arel.sql("(#{d_rel.to_sql}) as mochigome_data")
+              )
+              d_tbl = Arel::Table.new("mochigome_data")
+              agg_fields.each_with_index do |a, i|
+                a_rel.project(a[:agg_proc].call(d_tbl["d#{i}"]))
+              end
+              key_cols.take(n).each_with_index do |col, i|
+                outer_name = "og#{i}"
+                a_rel.project(d_tbl["g#{i}"].as(outer_name)).group(outer_name)
+              end
+              a_rel
+            }
+          }
         end
-        cur_layer = new_layer
+
       end
+    end
 
-      # Take our tree so far and include it in parent trees, going up to the first layer
-      upwards_layers = @layer_types.take_while{|cls| !objs.first.is_a?(cls)}.reverse
-      upwards_layers.each do |cls|
-        assoc = Query.edge_assoc(root.children.first[:obj].class, cls)
+    def run(cond = nil)
+      root = DataNode.new(:report, @name)
 
-        assoc_str =  "<- #{cls.name} via #{root.children.first[:obj].class.name}##{assoc.name}"
-        if assoc.through_reflection
-          assoc_str << " (thru #{assoc.through_reflection.name})"
+      if cond.is_a?(ActiveRecord::Base)
+        cond = [cond]
+      end
+      if cond.is_a?(Array)
+        return root if cond.empty?
+        unless cond.all?{|obj| obj.class == cond.first.class}
+          raise QueryError.new("Query target objects must all be the same type")
         end
-        assoc_path.unshift assoc_str
-
-        parent_children_map = ActiveSupport::OrderedHash.new
-        root.children.each do |child|
-          if assoc.through_reflection
-            through_objs = child[:obj].send(assoc.through_reflection.name)
-            through_objs = [through_objs] unless through_objs.is_a?(Enumerable)
-            through_objs.each do |through_obj|
-              # TODO: Don't assume that through means singular!
-              parent = through_obj.send(assoc.source_reflection.name)
-              unless parent_children_map.has_key?(parent.id)
-                attrs = {:obj => parent, :through_obj => through_obj}
-                parent_children_map[parent.id] = DataNode.new(
-                  parent.mochigome_focus.type_name,
-                  parent.mochigome_focus.name,
-                  attrs
-                )
-              end
-              parent_children_map[parent.id] << child.dup
-            end
-          else
-            #FIXME: Not DRY
-            parents = child[:obj].send(assoc.name)
-            parents = [parents] unless parents.is_a?(Enumerable)
-            parents.each do |parent|
-              unless parent_children_map.has_key?(parent.id)
-                attrs = {:obj => parent}
-                parent_children_map[parent.id] = DataNode.new(
-                  parent.mochigome_focus.name,
-                  parent.mochigome_focus.type_name,
-                  attrs
-                )
+        unless @layer_types.any?{|layer| cond.first.is_a?(layer)}
+          raise QueryError.new("Query target's class must be in layer list")
+        end
+        cls = cond.first.class
+        cond = Arel::Table.new(cls.table_name)[cls.primary_key].in(cond.map(&:id))
+      end
+
+      q = @ids_rel.dup
+      q.where(cond) if cond
+      ids_table = @layer_types.first.connection.select_rows(q.to_sql)
+      ids_table = ids_table.map{|row| row.map{|cell| cell.to_i}}
+
+      fill_layers(ids_table, {:root => root}, @layer_types)
+
+      @aggregate_rels.each do |focus_model, data_model_rels|
+        super_types = @layer_types.take_while{|m| m != focus_model}
+        super_cols = super_types.map{|m| @layers_path.find_index(m)}
+        data_model_rels.each do |data_model, rel_funcs|
+          aggs = data_model.mochigome_aggregation_settings.options[:fields]
+          aggs_count = aggs.reject{|a| a[:in_ruby]}.size
+          rel_funcs.each do |rel_func|
+            q = rel_func.call(cond)
+            data_tree = {}
+            # Each row has aggs_count data fields, followed by the id fields
+            # from least specific to most.
+            @layer_types.first.connection.select_rows(q.to_sql).each do |row|
+              if row.size == aggs_count
+                data_tree = row.take(aggs_count)
+              else
+                c = data_tree
+                super_cols.each_with_index do |sc_num, sc_idx|
+                  break if aggs_count+sc_idx >= row.size-1
+                  col_num = aggs_count + super_cols[sc_num]
+                  c = (c[row[col_num].to_i] ||= {})
+                end
+                c[row.last.to_i] = row.take(aggs_count)
               end
-              parent_children_map[parent.id] << child.dup
             end
+            insert_aggregate_data_fields(root, data_tree, data_model)
           end
         end
-
-        root = DataNode.new(:report, @name)
-        root << parent_children_map.values
       end
 
       root.comment = <<-eos
         Mochigome Version: #{Mochigome::VERSION}
-        Time: #{Time.now}
+        Report Generated: #{Time.now}
         Layers: #{@layer_types.map(&:name).join(" => ")}
-        AR Association Path:
-        #{assoc_path.map{|s| "* #{s}"}.join("\n")}
+        AR Path: #{@layers_path.map(&:name).join(" => ")}
       eos
       root.comment.gsub!(/\n +/, "\n")
       root.comment.lstrip!
 
-      focus_data_node_objs(root)
       return root
     end
 
     private
 
-    def focus_data_node_objs(node, obj_stack=[], commenting=true)
-      pushed = 0
-      if node.has_key?(:obj)
-        obj = node.delete(:obj)
-        if node.has_key?(:through_obj)
-          obj_stack.push(node.delete(:through_obj)); pushed += 1
+    def access_filtered_relation(r, models)
+      joined = Set.new
+      models.uniq.each do |model|
+        h = @access_filter.call(model)
+        h.delete(:join_paths).try :each do |path|
+          (0..(path.size-2)).each do |i|
+            next if models.include?(path[i+1]) or joined.include?(path[i+1])
+            r = self.class.relation_func(path[i], path[i+1]).call(r)
+            joined.add path[i+1]
+          end
         end
-        obj_stack.push(obj); pushed += 1
-        if commenting
-          node.comment = <<-eos
-            Context:
-            #{obj_stack.map{|o| "#{o.class.name}:#{o.id}"}.join("\n")}
-          eos
-          node.comment.gsub!(/\n +/, "\n")
-          node.comment.lstrip!
+        if h[:condition]
+          r = r.where(h.delete(:condition))
+        end
+        unless h.empty?
+          raise QueryError.new("Unknown assoc filter keys #{h.keys.inspect}")
         end
-        node.merge!(obj.mochigome_focus.data(:context => obj_stack))
-        node[:internal_type] = obj.class.name
-      end
-      node.children.each_index do |i|
-        focus_data_node_objs(node.children[i], obj_stack, i == 0 && commenting)
       end
-      pushed.times{ obj_stack.pop }
+      r
     end
 
-    @@assoc_graph = nil
-    @@edge_assocs = {}
-
-    def self.assoc_graph
-      return @@assoc_graph if @assoc_graph
-
-      # Build a directed graph of the associations between focusable models
-      @@assoc_graph = RGL::DirectedAdjacencyGraph.new
-      @@assoc_graph.add_vertices(*Mochigome::reportFocusModels)
-      Mochigome::reportFocusModels.each do |cls|
-        # Add any associations that lead to other reportFocusModels
-        cls.reflections.each do |name, assoc|
-          if Mochigome::reportFocusModels.include?(assoc.klass)
-            @@assoc_graph.add_edge(cls, assoc.klass)
-            @@edge_assocs[[cls, assoc.klass]] = assoc
+    def fill_layers(ids_table, parents, types, parent_col_num = nil)
+      return if types.size == 0
+
+      model = types.first
+      col_num = @layers_path.find_index(model)
+      layer_ids = Set.new
+      cur_to_parent = {}
+
+      ids_table.each do |row|
+        cur_id = row[col_num]
+        layer_ids.add cur_id
+        if parent_col_num
+          cur_to_parent[cur_id] ||= Set.new
+          cur_to_parent[cur_id].add row[parent_col_num]
+        end
+      end
+
+      layer = {}
+      model.all( # TODO: Find a way to do this with data streaming
+        :conditions => {model.primary_key => layer_ids.to_a},
+        :order => model.mochigome_focus_settings.get_ordering
+      ).each do |obj|
+        f = obj.mochigome_focus
+        dn = DataNode.new(f.type_name, f.name)
+        dn.merge!(f.field_data)
+        # TODO: Maybe make special fields below part of ModelExtensions?
+        dn[:id] = obj.id
+        dn[:internal_type] = obj.class.name
+
+        if parent_col_num
+          duping = false
+          cur_to_parent.fetch(obj.id).each do |parent_id|
+            parents.fetch(parent_id) << (duping ? dn.dup : dn)
+            duping = true
           end
+        else
+          parents[:root] << dn
         end
+        layer[obj.id] = dn
+      end
+
+      fill_layers(ids_table, layer, types.drop(1), col_num)
+    end
+
+    def insert_aggregate_data_fields(node, table, data_model)
+      if table.is_a? Array
+        fields = data_model.mochigome_aggregation_settings.options[:fields]
+        # Pre-fill the node with all fields in the right order
+        fields.each{|agg| node[agg[:name]] = nil unless agg[:hidden] }
+        agg_row = {} # Hold regular aggs here to be used in ruby-based aggs
+        fields.reject{|agg| agg[:in_ruby]}.zip(table).each do |agg, v|
+          agg_row[agg[:name]] = v
+          node[agg[:name]] = v unless agg[:hidden]
+        end
+        fields.select{|agg| agg[:in_ruby]}.each do |agg|
+          node[agg[:name]] = agg[:ruby_proc].call(agg_row)
+        end
+      else
+        node.children.each do |c|
+          subtable = table[c[:id]] or next
+          insert_aggregate_data_fields(c, subtable, data_model)
+        end
+      end
+    end
+
+    # TODO: Move the stuff below into its own module
+
+    @@graphed_models = Set.new
+    @@assoc_graph = RGL::DirectedAdjacencyGraph.new
+    @@edge_relation_funcs = {}
+    @@shortest_paths = {}
+
+    def self.relation_over_path(path, rel = nil)
+      rel ||= Arel::Table.new(path.first.table_name)
+      (0..(path.size-2)).each do |i|
+        rel = relation_func(path[i], path[i+1]).call(rel)
       end
-      return @@assoc_graph
+      rel
     end
 
-    def self.edge_assoc(u, v)
-      assoc_graph # Make sure @@edge_assocs has been populated
-      assoc = @@edge_assocs[[u,v]]
-      unless assoc
-        raise QueryError.new("No association between #{u} and #{v}")
+    def self.relation_func(u, v)
+      @@edge_relation_funcs[[u,v]] or
+        raise QueryError.new "No assoc from #{u.name} to #{v.name}"
+    end
+
+    def self.path_thru(models)
+      update_assoc_graph(models)
+      path = [models.first]
+      (0..(models.size-2)).each do |i|
+        u = models[i]
+        v = models[i+1]
+        next if u == v
+        seg = @@shortest_paths[[u,v]]
+        raise QueryError.new("Can't travel from #{u.name} to #{v.name}") unless seg
+        seg.drop(1).each{|step| path << step}
+      end
+      unless path.uniq.size == path.size
+        raise QueryError.new(
+          "Path thru #{models.map(&:name).join('-')} doubles back: " +
+          path.map(&:name).join('-')
+        )
+      end
+      path
+    end
+
+    def self.update_assoc_graph(models)
+      model_queue = models.dup
+      added_models = []
+      until model_queue.empty?
+        model = model_queue.shift
+        next if @@graphed_models.include? model
+        @@graphed_models.add model
+        added_models << model
+
+        model.reflections.
+        reject{|name, assoc| assoc.through_reflection}.
+        each do |name, assoc|
+          # TODO: What about self associations?
+          # TODO: What about associations to the same model on different keys?
+          next if assoc.options[:polymorphic] # TODO How to deal with these? Check for matching has_X assoc?
+          foreign_model = assoc.klass
+          edge = [model, foreign_model]
+          next if @@assoc_graph.has_edge?(*edge) # Ignore duplicate assocs
+          @@assoc_graph.add_edge(*edge)
+          @@edge_relation_funcs[edge] = model.arelified_assoc(name)
+          unless @@graphed_models.include?(foreign_model)
+            model_queue.push(foreign_model)
+          end
+        end
+      end
+
+      added_models.each do |model|
+        next unless @@assoc_graph.has_vertex?(model)
+        path_tree = @@assoc_graph.bfs_search_tree_from(model).reverse
+        path_tree.depth_first_search do |tgt_model|
+          next if tgt_model == model
+          path = [tgt_model]
+          while (parent = path_tree.adjacent_vertices(path.first).first)
+            path.unshift parent
+          end
+          @@shortest_paths[[model,tgt_model]] = path
+        end
+
+        # Use through reflections as a hint for preferred indirect paths
+        model.reflections.
+        select{|name, assoc| assoc.through_reflection}.
+        each do |name, assoc|
+          begin
+            foreign_model = assoc.klass
+            join_model = assoc.through_reflection.klass
+          rescue NameError
+            # FIXME Can't handle polymorphic through reflection
+          end
+          edge = [model,foreign_model]
+          next if @@shortest_paths[edge].try(:size).try(:<, 3)
+          @@shortest_paths[edge] = [model, join_model, foreign_model]
+        end
       end
-      return assoc
     end
   end
 end

data/test/app_root/app/models/category.rb

@@ -1,7 +1,9 @@
 class Category < ActiveRecord::Base
-  acts_as_mochigome_focus
+  acts_as_mochigome_focus do |f|
+    f.ordering :name
+  end
 
-  has_many :products, :conditions => {:categorized => true}
+  has_many :products
 
   validates_presence_of :name
 end

data/test/app_root/app/models/product.rb

@@ -2,11 +2,20 @@ class Product < ActiveRecord::Base
   acts_as_mochigome_focus do |f|
     f.fields [:price]
   end
-  has_mochigome_aggregations [
-    :average_price,
-    {"Power level" => "9000+1"},
-    {"Expensive products" => [:count, "price > 40"]}
-  ]
+  has_mochigome_aggregations do |a|
+    a.fields [
+      :sum_price,
+      {"Expensive products" => [
+        :count,
+        Mochigome::null_unless(
+          lambda{|v| v.gt(10.00)},
+          lambda{|t| t[:price]}
+        )
+      ]}
+    ]
+    a.hidden_fields [ {"Secret count" => :count} ]
+    a.fields_in_ruby [ {"Count squared" => lambda{|row| row["Secret count"]**2}} ]
+  end
 
   belongs_to :category
   has_many :store_products

data/test/app_root/app/models/sale.rb

@@ -1,5 +1,7 @@
 class Sale < ActiveRecord::Base
-  has_mochigome_aggregations [:count]
+  has_mochigome_aggregations do |a|
+    a.fields [:count]
+  end
 
   belongs_to :store_product
   has_one :store, :through => :store_product

data/test/app_root/config/initializers/arel.rb

@@ -0,0 +1,2 @@
+require 'arel'
+Arel::Table.engine = Arel::Sql::Engine.new(ActiveRecord::Base)

data/test/app_root/db/migrate/20110817163830_create_tables.rb

@@ -4,7 +4,6 @@ class CreateTables < ActiveRecord::Migration
       t.string :name
       t.decimal :price
       t.integer :category_id
-      t.boolean :categorized
       t.timestamps
     end
 

data/test/console.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+SCRIPT=`readlink -f $0`
+SCRIPTPATH=`dirname $SCRIPT`
+cd $SCRIPTPATH
+export NO_MINITEST=true
+irb -r test_helper

data/test/factories.rb

@@ -4,7 +4,6 @@ FactoryGirl.define do
   factory :product do
     name 'LG Optimus V'
     price 19.95
-    categorized true
   end
 
   factory :category do
@@ -23,7 +22,7 @@ FactoryGirl.define do
     phone_number "800-555-1212"
     email_address { "#{first_name}.#{last_name}@example.com".downcase }
   end
-  
+
   factory :store_product do
     store
     product