activewarehouse 0.2.0 → 0.3.0

data/lib/active_warehouse/aggregate/dwarf_printer.rb

@@ -0,0 +1,34 @@
+module ActiveWarehouse #:nodoc:
+  module Aggregate #:nodoc:
+    # Dwarf support class that prints a representation of the Dwarf
+    class DwarfPrinter
+      # Print the specified node at the given depth.
+      def self.print_node(node, depth=0, recurse=true)
+        #puts "printing node #{node.index}"
+        cells = node.cells.collect { |c| cell_to_string(c)}.join('|')
+
+        parent_node = node.parent ? "#{cell_to_string(node.parent)}:" : ''
+        puts "#{node.index}=#{' '*depth}#{parent_node}[#{cells}|#{all_cell_to_string(node.all_cell)}]"
+        if !node.leaf?
+          print_node(node.all_cell.child, depth + 1, false) if node.all_cell
+        end
+        if recurse
+          node.children.each { |child| print_node(child, depth+1) }
+        end
+      end
+      
+      def self.cell_to_string(cell)
+        # a new String object must be created here, otherwise to_s returns a reference
+        # to the same String object each time and thus the value will be appended each time
+        # which is not what I want
+        s = String.new(cell.key.to_s)
+        s << " #{cell.value.join(',')}" if cell.node.leaf?
+        s
+      end
+      
+      def self.all_cell_to_string(cell)
+        cell ? (cell.value ? cell.value.inspect : '') : ''
+      end
+    end
+  end
+end

data/lib/active_warehouse/aggregate/no_aggregate.rb

@@ -0,0 +1,194 @@
+require 'set'
+
+module ActiveWarehouse #:nodoc:
+  module Aggregate #:nodoc:
+    # An aggregate which goes directly to the fact and dimensions to answer questions
+    class NoAggregate < Aggregate
+      # Populate the aggregate (in this case it is a no-op implementation)
+      def populate
+        # do nothing
+      end
+      
+      # Query the aggregate
+      # def query(column_dimension_name, column_hierarchy_name,
+#                 row_dimension_name, row_hierarchy_name, conditions=nil,
+#                 cstage=0, rstage=0, filters={})
+      
+      # Query the aggregate
+      def query(*args)
+        options = parse_query_args(*args)
+        
+        column_dimension_name = options[:column_dimension_name]
+        column_hierarchy_name = options[:column_hierarchy_name]
+        row_dimension_name = options[:row_dimension_name]
+        row_hierarchy_name = options[:row_hierarchy_name]
+        conditions = options[:conditions]
+        cstage = options[:cstage] || 0
+        rstage = options[:rstage] || 0
+        filters = options[:filters] || {}
+        
+        fact_class = cube_class.fact_class
+        column_dimension = fact_class.dimension_class(column_dimension_name)
+        column_hierarchy = column_dimension.hierarchy(column_hierarchy_name)
+        row_dimension = fact_class.dimension_class(row_dimension_name)
+        row_hierarchy = row_dimension.hierarchy(row_hierarchy_name)
+        
+        used_dimensions = Set.new
+        used_dimensions.merge([column_dimension_name, row_dimension_name])
+        row_dim_reflection = fact_class.dimension_relationships[row_dimension_name].dependent_dimension_reflections
+        used_dimensions.merge(row_dim_reflection.collect{|d| d.name})
+        col_dim_reflection = fact_class.dimension_relationships[column_dimension_name].dependent_dimension_reflections
+        used_dimensions.merge(col_dim_reflection.collect{|d| d.name})
+        filters.each do |k,v|
+          used_dimensions << k.split('.')[0]
+        end
+        if conditions
+          cube_class.dimensions.each do |dimension|
+            if conditions =~ /#{dimension}\./i
+              used_dimensions << dimension
+            end
+          end
+        end
+        
+        # This method assumes at most one dimension is hierarchical dimension
+        # in the query params. TODO: need to handle when both row and column
+        # are hierarchical dimensions.
+        hierarchical_dimension = nil
+        hierarchical_dimension_name = nil
+        hierarchical_stage = nil
+        
+        if !column_dimension.hierarchical_dimension?
+          current_column_name = column_hierarchy[cstage]
+        else
+          hierarchical_dimension = column_dimension
+          hierarchical_dimension_name = column_dimension_name
+          hierarchical_stage = cstage
+          current_column_name = column_hierarchy[0]
+        end
+        
+        if !row_dimension.hierarchical_dimension?
+          current_row_name = row_hierarchy[rstage]
+        else
+          hierarchical_dimension = row_dimension
+          hierarchical_dimension_name = row_dimension_name
+          hierarchical_stage = rstage
+          current_row_name = row_hierarchy[0]
+        end
+
+        fact_columns = cube_class.aggregate_fields.collect { |c| 
+          agg_sql = ''
+          quoted_label = cube_class.connection.quote_column_name(c.label)
+          if hierarchical_dimension and !c.levels_from_parent.empty?
+            bridge = hierarchical_dimension.bridge_class
+            bridge_table_name = bridge.table_name
+            levels_from_parent = bridge.levels_from_parent
+            get_all = false
+            c.levels_from_parent.each do |level|
+              case level
+              when :all
+                agg_sql += "  #{c.strategy_name}(#{c.from_table_name}.#{c.name}) AS #{quoted_label})"
+                get_all = true
+              when :self
+                agg_sql += " #{c.strategy_name}(CASE " if agg_sql.length == 0
+                agg_sql += " WHEN #{bridge_table_name}.#{levels_from_parent} = 0 THEN #{c.from_table_name}.#{c.name} \n"
+              when Integer  
+                agg_sql += " #{c.strategy_name}(CASE " if agg_sql.length == 0              
+                agg_sql += " WHEN #{bridge_table_name}.#{levels_from_parent} = #{level} then #{c.from_table_name}.#{c.name} \n"
+              else
+                raise ArgumentError, "Each element to :levels_from_parent option must be :all, :self, or Integer"
+              end
+            end
+            agg_sql += " ELSE 0 END) AS #{quoted_label}" unless get_all
+          else
+            agg_sql = "  #{c.strategy_name}(#{c.from_table_name}.#{c.name}) AS #{quoted_label}" 
+          end
+          agg_sql
+        }.join(",\n")
+
+        sql = ''
+        sql += "SELECT\n"
+        sql += "  #{column_dimension_name}.#{current_column_name},\n"
+        sql += "  #{row_dimension_name}.#{current_row_name},\n"
+        sql += fact_columns
+        sql += "\nFROM\n"
+
+        sql += "  #{fact_class.table_name}"
+        cube_class.dimensions_hierarchies.each do |dimension_name, hierarchy_names|
+          next if !used_dimensions.include?(dimension_name)
+          dimension = fact_class.dimension_class(dimension_name)
+          if !dimension.hierarchical_dimension?
+            sql += "\nJOIN #{dimension.table_name} as #{dimension_name}"
+            sql += "\n  ON #{fact_class.table_name}.#{fact_class.foreign_key_for(dimension_name)} = "
+            sql += "#{dimension_name}.#{dimension.primary_key}"
+          else
+            dimension_bridge = dimension.bridge_class
+            sql += "\nJOIN #{dimension_bridge.table_name}"
+            sql += "\n  ON #{fact_class.table_name}.#{fact_class.foreign_key_for(dimension_name)} = "
+            sql += "#{dimension_bridge.table_name}.#{dimension.parent_foreign_key}"
+            if dimension.slowly_changing_dimension?
+              sql += " and (#{dimension_bridge.table_name}.#{dimension_bridge.effective_date} <= "
+              sql += "#{fact_class.slowly_changes_over_name(dimension_name)}."
+              sql += "#{fact_class.slowly_changes_over_class(dimension_name).sql_date_stamp} "
+              sql += "and #{dimension_bridge.table_name}.#{dimension_bridge.expiration_date} >= "
+              sql += "#{fact_class.slowly_changes_over_name(dimension_name)}."
+              sql += "#{fact_class.slowly_changes_over_class(dimension_name).sql_date_stamp}) "
+            end
+            sql += "\nJOIN #{dimension.table_name} as #{dimension_name}"
+            sql += "\n  ON #{dimension_bridge.table_name}.#{dimension.child_foreign_key} = "
+            sql += "#{dimension_name}.#{dimension.primary_key}"
+          end
+        end
+
+        # build the where clause
+        # first add conditions
+        where_clause = Array(conditions)
+        
+        # apply filters
+        filters.each do |key, value|
+          dimension_name, column = key.split('.')
+          where_clause << "#{dimension_name}.#{column} = #{cube_class.connection.quote(value)}"
+        end
+        sql += %Q(\nWHERE\n  #{where_clause.join(" AND\n  ")} ) if where_clause.length > 0
+
+        # for hierarchical dimension we need to add where clause in for drill downs
+        if !hierarchical_dimension.nil?
+          if where_clause.length == 0
+            sql += "\n WHERE "
+          else 
+            sql += " \n AND "
+          end
+          sql += "\n #{hierarchical_dimension_name}.#{hierarchical_dimension.primary_key} IN ( "
+          sql += "\n SELECT #{hierarchical_dimension.parent_foreign_key} FROM #{hierarchical_dimension.bridge_class.table_name} "
+          if hierarchical_stage == 0   
+            sql += "\n WHERE #{hierarchical_dimension.bridge_class.top_flag} = #{connection.send(:quote, hierarchical_dimension.bridge_class.top_flag_value)})"
+          else
+            sql += "\n WHERE #{hierarchical_dimension.child_foreign_key} = #{hierarchical_stage} AND #{hierarchical_dimension.levels_from_parent} = 1)"
+          end
+        end
+        
+        sql += "\nGROUP BY\n"
+        sql += "  #{column_dimension_name}.#{current_column_name},\n"
+        sql += "  #{row_dimension_name}.#{current_row_name}"
+        
+        if options[:order]
+          order_by = options[:order]
+          order_by = [order_by] if order_by.is_a?(String)
+          order_by.collect!{ |v| cube_class.connection.quote_column_name(order_by) }
+          sql += %Q(\nORDER BY\n  #{order_by.join(",\n")})
+        end
+        
+        result = ActiveWarehouse::CubeQueryResult.new(
+          cube_class.aggregate_fields
+        )
+        
+        cube_class.connection.select_all(sql).each do |row|
+          result.add_data(row.delete(current_row_name.to_s),
+                          row.delete(current_column_name.to_s),
+                          row) # the rest of the members of row are the fact columns
+        end
+        
+        result
+      end
+    end
+  end
+end

data/lib/active_warehouse/aggregate/pid_aggregate.rb

@@ -0,0 +1,29 @@
+module ActiveWarehouse #:nodoc:
+  module Aggregate #:nodoc:
+    # Implementation of a Partitioning and Inserting Dwarf algorithm as defined
+    # in http://www.zju.edu.cn/jzus/2005/A0506/A050608.pdf
+    class PidAggregate < Aggregate
+      include DwarfCommon
+      
+      # Initialize the aggregate
+      def initialize(cube_class)
+        super
+      end
+      
+      # Populate the aggregate
+      def populate
+        create_dwarf_cube(sorted_facts)
+      end
+      
+      # Query the aggregate
+      def query(*args)
+        options = parse_query_args(*args)
+      end
+      
+      def create_dwarf_cube(sorted_facts)
+        
+      end
+
+    end
+  end
+end

data/lib/active_warehouse/aggregate/pipelined_rolap_aggregate.rb

@@ -0,0 +1,129 @@
+require 'erb'
+
+module ActiveWarehouse #:nodoc
+  module Aggregate #:nodoc
+    # A Pipelined implementation of a ROLAP engine that stores all possible
+    # combinations
+    # of fact and dimensional values for a specific cube.
+    # 
+    # This implementation attempts to reduce the amount of work required
+    # by aggregating facts in a pipelined fashion.  This means that smaller
+    # aggregates are generated from a preceding aggregate, in order to avoid
+    # having to query the entire raw data set for every aggregate.
+    # 
+    # E.g.
+    # 
+    # ABCD -> ABC -> AB -> A -> *all*
+    class PipelinedRolapAggregate < Aggregate
+      include RolapCommon
+
+      # Build and populate the data store
+      def populate(options={})
+        create_and_populate_all_table
+        create_all_pipelined_agg_tables
+        create_insert_statements.each_with_index do |insert, i|
+          next if i == 0 #handled by create_and_populate_all_table
+          connection.transaction {connection.execute(insert)}
+        end
+      end
+      
+      protected
+      
+      # build and populate a table which group by's all dimension columns.
+      def create_and_populate_all_table
+        dimension_column_names = dimensions_to_columns.collect do |c|
+          "#{c.table_alias}.#{c.name} as #{c.table_alias}_#{c.name}"
+        end
+        
+        fact_column_names = aggregate_fields.collect do |c|
+          "#{c.from_table_name}.#{c.name} as #{c.label_for_table}"
+        end
+        
+        sql = <<-SQL
+          SELECT
+          #{dimension_column_names.join(",")},
+          #{aggregated_fact_column_sql_for_all}
+          FROM #{tables_and_joins}
+          GROUP BY
+          #{dimensions_to_columns.collect{|c| "#{c.table_alias}.#{c.name}"}.join(",")}
+        SQL
+        
+        all_table_name = indexed_rollup_table_name(dimension_column_names.length)
+        
+        sql = connection.add_select_into_table(all_table_name, sql)
+        
+        connection.drop_table(all_table_name) if connection.tables.include?(all_table_name)
+        connection.transaction { connection.execute(sql) }
+      end
+      
+      def create_all_pipelined_agg_tables
+        (0..dimensions_to_columns.size-1).each do |i|
+          create_rollup_cube_table(i)
+        end
+      end
+      
+      # Creates the rollup table
+      def create_rollup_cube_table(index)
+        table_name = indexed_rollup_table_name(index)
+        connection.drop_table(table_name) if connection.tables.include?(table_name)
+        
+        ActiveRecord::Base.transaction do
+          connection.create_table(table_name, :id => false) do |t|
+            dimensions_to_columns.each do |c|
+              t.column(c.label, c.column_type)
+            end
+            aggregate_fields.each do |c|
+              options = {}
+              options[:limit] = c.column_type == :integer ? 8 : c.limit
+              options[:scale] = c.scale if c.scale
+              options[:precision] = c.precision if c.precision
+              t.column(c.label_for_table, c.column_type, options)
+            end
+          end
+        end
+      end
+      
+      def create_insert_statements
+        dim_columns = dimensions_to_columns
+        template_filename = File.dirname(__FILE__) + "/templates/pipelined_rollup_#{dim_columns.length}.sql"
+        dim_columns.length.times do |i|
+          eval("@dimension_#{i} = '#{dim_columns[i].label}'")
+        end
+        @aggregate_fields_from_flat_table = aggregated_fact_column_sql_for_rollup
+        @aggregate_fields = aggregated_fact_column_sql_for_rollup
+        @flat_table_name = flat_table_name
+        @rollup_table_name = rollup_table_name
+        
+        inserts = []
+        
+        sql = ""
+        ERB.new(File.read(template_filename)).result(binding).each do |line|
+          if line.strip == ""
+            inserts << sql
+            sql = ""
+          else
+            sql += line
+          end
+        end
+        
+        inserts
+      end
+      
+      def indexed_rollup_table_name(index)
+        "#{rollup_table_name}_#{index}"
+      end
+      
+      def aggregated_fact_column_sql_for_all
+        aggregate_fields.collect { |c| 
+          "#{c.strategy_name}(#{c.name}) AS #{c.label_for_table}"
+        }.join(",")
+      end
+      
+      def aggregated_fact_column_sql_for_rollup
+        aggregate_fields.collect { |c|
+          "#{c.strategy_name == :avg ? :avg : :sum}(#{c.label_for_table}) AS #{c.label_for_table}"
+        }.join(", ")
+      end
+    end
+  end
+end

data/lib/active_warehouse/aggregate/rolap_aggregate.rb

@@ -0,0 +1,181 @@
+# Source file that contains a basic ROLAP engine implementation.
+
+module ActiveWarehouse #:nodoc
+  module Aggregate #:nodoc
+    # Basic implementation of a ROLAP engine that stores all possible combinations
+    # of fact and dimensional values for a specific cube.
+    class RolapAggregate < Aggregate
+      include RolapCommon
+    
+      # Build and populate the data store
+      def populate(options={})
+        populate_rollup_cube
+      end
+      
+      # Query the aggregate, returning a QueryResult object
+      def query(*args)
+        options = parse_query_args(*args)
+        
+        column_dimension_name = options[:column_dimension_name]
+        column_hierarchy_name = options[:column_hierarchy_name]
+        row_dimension_name = options[:row_dimension_name]
+        row_hierarchy_name = options[:row_hierarchy_name]
+        conditions = options[:conditions]
+        cstage = options[:cstage]
+        rstage = options[:rstage]
+        filters = options[:filters]
+        
+        column_dimension = fact_class.dimension_class(column_dimension_name)
+        column_hierarchy = column_dimension.hierarchy(column_hierarchy_name)
+        row_dimension = fact_class.dimension_class(row_dimension_name)
+        row_hierarchy = row_dimension.hierarchy(row_hierarchy_name)
+        current_column_name = column_hierarchy[cstage]
+        current_row_name = row_hierarchy[rstage]
+        full_column_name = "#{column_dimension_name}_#{current_column_name}"
+        full_row_name = "#{row_dimension_name}_#{current_row_name}"
+
+        # build the SQL query
+        sql = ''
+        sql += 'SELECT '
+        sql += "#{full_column_name} AS #{current_column_name},"
+        sql += "#{full_row_name} AS #{current_row_name},"
+        sql += aggregate_fields.collect{|c| "#{c.label_for_table} as '#{c.label}'"}.join(",")
+        sql += " FROM #{rollup_table_name} "
+
+        # build the where clause
+        where_clause = []
+        0.upto(column_hierarchy.length - 1) do |stage|
+          column_name = column_hierarchy[stage]
+          name = "#{column_dimension_name}_#{column_name}"
+          filter_value = filters.delete(column_name)
+          if filter_value
+            where_clause << "#{name} = '#{filter_value}'" # TODO: protect from
+                                                          # SQL injection
+          else
+            where_clause << "#{name} is null" if stage > cstage
+          end
+        end
+        0.upto(row_hierarchy.length - 1) do |stage|
+          row_name = row_hierarchy[stage]
+          name = "#{row_dimension_name}_#{row_name}"
+          filter_value = filters.delete(row_name)
+          if filter_value
+            where_clause << "#{name} = '#{filter_value}'" # TODO: protect from
+                                                          # SQL injection
+          else
+            where_clause << "#{name} is null" if stage > rstage
+          end
+        end
+        where_clause << "#{full_column_name} is not null"
+        where_clause << "#{full_row_name} is not null"
+        filters.each do |key, value|
+          dimension_name, column = key.split('.')
+          where_clause << "#{dimension_name}_#{column} = '#{value}'" # TODO: protect from SQL injection
+        end
+        sql += %Q( WHERE #{where_clause.join(" AND ")} ) if where_clause.length > 0
+        
+         if conditions
+            sql += "\n WHERE\n" unless sql =~ /WHERE/i
+            sql += conditions
+          end
+        
+        # execute the query and return the results as a CubeQueryResult object
+        result = ActiveWarehouse::CubeQueryResult.new(
+          aggregate_fields
+        )
+        rows = connection.select_all(sql)
+#        fact_column_names = fact_class.aggregate_fields.collect{|f| f.to_s}
+        rows.each do |row|
+          result.add_data(row.delete(current_row_name.to_s),
+                          row.delete(current_column_name.to_s),
+                          row) # the rest of the members of row are the fact columns
+        end
+        result
+      end
+      
+      protected
+      
+      # Creates the rollup table
+      def create_rollup_cube_table(options={})
+        # TODO: perhaps this should all be executed in a single transaction?
+        connection.drop_table(rollup_table_name) if connection.tables.include?(rollup_table_name)
+        
+        ActiveRecord::Base.transaction do
+          connection.create_table(rollup_table_name, :id => false) do |t|
+            dimensions_to_columns.each do |c|
+              t.column(c.label, c.column_type)
+            end
+            aggregate_fields.each do |c|
+              t.column(c.label_for_table, c.column_type)
+            end
+          end
+        end
+      end
+      
+      # Builds the aggregate SQL that will be used to populate the ROLAP table.
+      # This SQL is just the SELECT statement and includes all of the GROUP BYs
+      # and aggregation functions.
+      # 
+      # +column_mask+ is an array of booleans, where true is the column to group
+      # by.  The length of this array is equal to the number of columns in
+      # the SELECT clause.
+      def build_aggregate_sql(column_mask)
+        dimension_column_names = dimensions_to_columns.collect do |c|
+          "#{c.table_alias}.#{c.name}"
+        end
+        
+        sql = <<-SQL
+          SELECT
+          #{mask_columns_with_null(dimension_column_names, column_mask).join(",")},
+          #{aggregated_fact_column_sql}
+          FROM #{tables_and_joins}
+        SQL
+        
+        group = mask_columns_with_null(dimension_column_names, column_mask).reject{|o| o == 'null'}.join(",")
+        sql += "GROUP BY #{group}" if !group.empty?
+        
+        sql
+      end
+
+      # Populate the rollup cube
+      # 
+      # Options:
+      # * <tt>:verbose</tt>: Set to true to print info to STDOUT during building
+      def populate_rollup_cube(options={})
+        create_rollup_cube_table(options)
+        puts "Populating rollup cube #{cube_class.name}" if options[:verbose]
+        
+        num_columns = dimensions_to_columns.size
+        num_combos = (2**num_columns)-1
+        puts "There are #{num_combos} combinations" if options[:verbose]
+        (0..num_combos).each do |i|
+          puts "Populating agg #{i} of #{num_combos}" if i % 100 == 0 if options[:verbose]
+          mask = sprintf("%0#{num_columns}b", i).split(//).collect{|x| x == '1' ? true : false}
+
+          sql = ''
+          sql += "INSERT INTO #{rollup_table_name} "
+          sql += build_aggregate_sql(mask)
+          
+          connection.transaction { connection.execute(sql) }
+        end
+        
+        if options[:verbose]
+          row_count = connection.select_value("SELECT count(*) FROM #{rollup_table_name}")
+          puts "Rollup cube populated with #{row_count} rows"
+        end
+      end
+      
+      # Mask columns with null 
+      def mask_columns_with_null(column_names, mask)
+        if mask.size != column_names.size
+          raise "Columns has #{column_names.size} elements, but mask has only #{mask.size}"
+        end
+
+        new_columns = []
+        column_names.each_with_index{ |c,i| new_columns << (mask[i] ? c : 'null')}
+        new_columns
+      end
+      
+    end
+  end
+end