activewarehouse 0.1.0

data/lib/active_warehouse/model.rb

@@ -0,0 +1,5 @@
+require 'active_warehouse/model/aggregate'
+require 'active_warehouse/model/fact'
+require 'active_warehouse/model/dimension'
+require 'active_warehouse/model/cube'
+require 'active_warehouse/model/report'

data/lib/active_warehouse/model/aggregate.rb

@@ -0,0 +1,244 @@
+module ActiveWarehouse
+  # An aggreate within a cube used to store calculated values
+  # Each aggregate will contain values for a dimension pair, down each of the dimension hierarchies
+  class Aggregate < ActiveRecord::Base
+    class << self
+      attr_accessor :name, :cube, :dimension1, :dimension2, :dimension1_hierarchy_name, :dimension2_hierarchy_name
+      
+      # Get the table name for the aggregate
+      def table_name
+        name = self.name.demodulize.underscore
+        set_table_name(name)
+        name
+      end
+      
+      # Returns the aggregate ID
+      def aggregate_id
+        table_name =~ /(\d+)$/
+        $1.to_i
+      end
+      
+      # Returns the AggregateMetaData instance associated with this aggregate
+      def meta_data
+        AggregateMetaData.find(aggregate_id)
+      end
+      
+      # Return true if the aggregate needs to be rebuilt
+      def needs_rebuild?(last_build=nil)
+        return true if meta_data.populated_at.nil?
+        return true if last_build && (meta_data.populated_at < last_build)
+        return false
+      end
+      
+      # Return a key for the aggregate
+      def key(dimension1, dimension1_hierarchy, dimension2, dimension2_hierarchy)
+        AggregateKey.new(dimension1, dimension1_hierarchy, dimension2, dimension2_hierarchy)
+      end
+      
+      # Create the aggregate table if required. Set force option to true to force creation of the table
+      # if it already exists
+      def create_storage_table(force=false)
+        connection.drop_table(table_name) if force and table_exists?
+        if !table_exists?
+          connection.create_table(table_name, :id => false) do |t|
+            t.column :dimension1_path, :string
+            t.column :dimension1_stage, :integer
+            t.column :dimension2_path, :string
+            t.column :dimension2_stage, :integer
+            cube.fact_class.aggregate_fields.each do |field|
+              #options = cube.fact_class.aggregate_field_options[field]
+              col = cube.fact_class.columns_hash[field.to_s]
+              t.column field, col.type if col
+            end
+          end
+          connection.add_index(table_name, :dimension1_path)
+          connection.add_index(table_name, :dimension1_stage)
+          connection.add_index(table_name, :dimension2_path)
+          connection.add_index(table_name, :dimension2_stage)
+        end
+      end
+      
+      # Populate the aggregate table
+      def populate
+        # create the storage table if necessary
+        create_storage_table
+        
+        #puts "Populating aggregate table #{table_name}"
+        # clear out the current data
+        #connection.execute("TRUNCATE TABLE #{table_name}") #TODO: make this generic to support all databases
+        delete_all
+        
+        # aggregate the data for the two dimensions
+        fact_class = cube.fact_class
+        dim1 = Dimension.class_name(dimension1).constantize
+        dim2 = Dimension.class_name(dimension2).constantize
+        dim1_stage_path = []
+        dim1.hierarchy(meta_data.dimension1_hierarchy.to_sym).each_with_index do |dim1_stage_name, dim1_stage_level|
+          dim1_stage_path << dim1_stage_name
+          dim2_stage_path = []
+          dim2.hierarchy(meta_data.dimension2_hierarchy.to_sym).each_with_index do |dim2_stage_name, dim2_stage_level|
+            dim2_stage_path << dim2_stage_name
+            
+            stmt, fields = build_query(fact_class, dim1, dim1_stage_path, dim2, dim2_stage_path)
+            
+            # Get the facts and aggregate them
+            fact_class.connection.execute(stmt).each do |row|
+              dim1_value = []
+              dim1_stage_path.each_with_index do |v, index|
+                dim1_value << row[index]
+              end
+              dim2_value = []
+              dim2_stage_path.each_with_index do |v, index|
+                dim2_value << row[dim1_stage_path.length + index]
+              end
+              
+              agg_instance = new
+              agg_instance.dimension1_path = dim1_value.join(':')
+              agg_instance.dimension1_stage = dim1_stage_level
+              agg_instance.dimension2_path = dim2_value.join(':')
+              agg_instance.dimension2_stage = dim2_stage_level 
+              fields.each_with_index do |field, index|
+                # do the average here
+                agg_instance.send("#{field}=".to_sym, row[index + dim1_value.length + dim2_value.length])
+              end
+              agg_instance.save!
+              
+              meta_data.update_attribute(:populated_at, Time.now)
+            end
+          end
+        end
+      end
+      
+      # Build the aggregation query for the given dimensions and stage paths
+      def build_query(fact_class, dim1, dim1_stage_path, dim2, dim2_stage_path)
+        dim1_group = dim1_stage_path.collect { |p| "d1.#{p}"}.join(", ")
+        dim2_group = dim2_stage_path.collect { |p| "d2.#{p}"}.join(", ")
+
+        # Set up the find options
+        fact_find_options = {}
+        fact_find_options[:group] = "#{dim1_group}, #{dim2_group}"
+        fact_find_options[:joins] = "join #{dim1.table_name} d1 on f.#{dim1.foreign_key} = d1.id"
+        fact_find_options[:joins] << " join #{dim2.table_name} d2 on f.#{dim2.foreign_key} = d2.id"
+
+        # Build the 'select' part of the query
+        # denominator = nil
+        fields = []
+        fact_select = ["#{dim1_group}, #{dim2_group}"]
+        fact_class.aggregate_fields.each do |field_name|
+          options = fact_class.aggregate_field_options[field_name]
+          fields << field_name
+          
+          options[:type] ||= :sum
+          case options[:type]
+          when :sum
+            fact_select << " sum(f.#{field_name}) as #{field_name}"
+          when Hash
+            if options[:type][dim1.sym] == :average && options[:type][dim2.sym] == :average
+              # I believe this is a special case, but I'm not sure how yet. If both dimensions are defined
+              # averages then perhaps that value cannot be calculated at all. TODO: research
+            else
+              fact_select << " sum(f.#{field_name}) as #{field_name}"
+            end
+          else
+            raise "Unsupported aggregate type: #{options[:type]}"
+          end
+        end
+        fact_find_options[:select] = fact_select.join(',')
+
+        # put the SQL statement together
+        stmt = "select #{fact_find_options[:select]} from "
+        stmt << "#{fact_class.table_name} f #{fact_find_options[:joins]} "
+        stmt << "group by #{fact_find_options[:group]}"
+
+        return stmt, fields
+      end
+      
+    end
+    
+    public
+    # Clone and reset at the same time
+    def clone_and_reset
+      o = clone
+      o.reset
+      o
+    end
+    
+    def non_data_fields
+      ['dimension1_path','dimension1_stage','dimension2_path','dimension2_stage']
+    end
+    
+    def data_fields
+      fields = []
+      self.class.columns.each do |column|
+        unless non_data_fields.include?(column.name)
+          fields << column.name
+        end
+      end
+      fields
+    end
+    
+    # Reset the aggregate
+    def reset
+      self.class.columns.each do |column|
+        unless non_data_fields.include?(column.name)
+          value = column.number? ? 0 : 'None'
+          send("#{column.name}=".to_sym, value)
+        end
+      end
+    end
+  end
+  
+  # ActiveRecord object which stores meta data about the aggregate
+  class AggregateMetaData < ActiveRecord::Base
+    # Build the underlying table. Set force to true to force the build of the table
+    def self.build_table(force=false)
+      connection.drop_table(table_name) if force and table_exists?
+      if !table_exists?
+        connection.create_table(table_name) do |t|
+          t.column :cube_name, :string
+          t.column :dimension1, :string
+          t.column :dimension1_hierarchy, :string
+          t.column :dimension2, :string
+          t.column :dimension2_hierarchy, :string
+          t.column :created_at, :datetime
+          t.column :populated_at, :datetime
+        end
+        connection.add_index table_name, :cube_name
+      end
+    end
+    def key
+      Aggregate.key(dimension1, dimension1_hierarchy, dimension2, dimension2_hierarchy)
+    end
+  end
+  
+  # Key for aggregate caching
+  class AggregateKey
+    attr_reader :dimension1, :dimension1_hierarchy, :dimension2, :dimension2_hierarchy
+    
+    def initialize(dimension1, dimension1_hierarchy, dimension2, dimension2_hierarchy)
+      @dimension1 = dimension1
+      @dimension1_hierarchy = dimension1_hierarchy
+      @dimension2 = dimension2
+      @dimension2_hierarchy = dimension2_hierarchy
+    end
+    
+    def ==(o)
+      o.instance_of?(self.class) and (o.to_s == to_s or o.to_s = to_rs)
+    end
+    
+    def hash
+      to_s.hash
+    end
+    
+    def to_s
+      "#{@dimension1}.#{@dimension1_hierarchy}.#{@dimension2}.#{@dimension2_hierarchy}"
+    end
+    
+    # Return the "reveresed" version of this key String representation
+    def to_rs
+      "#{@dimension2}.#{@dimension2_hierarchy}.#{@dimension1}.#{@dimension1_hierarchy}"
+    end
+  end
+end
+
+ActiveWarehouse::AggregateMetaData.build_table

data/lib/active_warehouse/model/cube.rb

@@ -0,0 +1,273 @@
+module ActiveWarehouse
+  # A Cube represents a collection of dimensions operating on a fact. The Cube provides a front-end for getting at the
+  # underlying data. The Cube manages the creation and population of all underlying aggregates.
+  class Cube
+    class << self
+      # Callback which is invoked when subclasses are created
+      def inherited(subclass)
+        subclasses << subclass
+      end
+      
+      # Get a list of all known subclasses
+      def subclasses
+        @subclasses ||= []
+      end
+      
+      # Defines the dimensions that this cube pivots on.
+      def pivots_on(*dimension_list)
+        # TODO: Validate if the fact is set
+        dimension_list.each do |dimension|
+          dimensions << dimension
+        end
+      end
+      
+      # Defines the fact that this cube reports on
+      def reports_on(fact)
+        # TODO: Validate if one or more dimension is set
+        @fact = fact
+      end
+      
+      # Rebuild all aggregate classes. Set :force => true to force the rebuild of aggregate classes.
+      def rebuild(options={})
+        logger.debug "Rebuilding aggregates for cube #{name}"
+        options[:force] ||= false
+        build_aggregate_classes(options)
+      end
+      
+      # Populate all aggregates. Set :force => true to force the population of the aggregate class.
+      def populate(options={})
+        options[:force] ||= false
+        aggregates.each do |agg_id, agg_clazz|
+          if agg_clazz.needs_rebuild? || options[:force]
+            logger.debug "Populating aggregate class #{agg_clazz.name}"
+            agg_clazz.populate
+          end
+        end
+      end
+      
+      # Get the fact that this cube reports on
+      def fact
+        @fact
+      end
+      
+      # Get the dimensions that this cube pivots on
+      def dimensions
+        @dimensions ||= []
+      end
+      
+      # Get the aggregate classes for this dimension
+      def aggregates
+        rebuild if @aggregates.nil?
+        @aggregates
+      end
+      
+      # Get the class name for the specified cube name
+      # Example: Regional Sales will become RegionalSalesCube
+      def class_name(name)
+        cube_name = name.to_s
+        cube_name = "#{cube_name}_cube" unless cube_name =~ /_cube$/
+        cube_name.classify
+      end
+      
+      # Get the aggregated fact class name
+      def fact_class_name
+        Fact.class_name(fact)
+      end
+      
+      # Get the aggregated fact class instance
+      def fact_class
+        fact_class_name.constantize
+      end
+      
+      # Get a list of dimension class instances
+      def dimension_classes
+        dimensions.collect {|dimension| Dimension.class_name(dimension).constantize} 
+      end
+      
+      def logger
+        @logger ||= Logger.new('cube.log')
+      end
+      
+      def last_modified
+        lm = Fact.class_for_name(fact).last_modified
+        dimensions.each do |dimension|
+          dim = Dimension.class_for_name(dimension)
+          lm = dim.last_modified if dim.last_modified > lm
+        end
+        lm
+      end
+      
+      protected
+      def build_aggregate_classes(options={})
+        @aggregates = {}
+        existing_dimension_pairs = []
+        logger.debug "Building aggregate classes"
+        dimensions.each do |column_dimension|
+          dimensions.each do |row_dimension|
+            next if column_dimension == row_dimension
+            next if existing_dimension_pairs.include? [column_dimension,row_dimension]
+            next if existing_dimension_pairs.include? [row_dimension,column_dimension]
+            
+            existing_dimension_pairs << [column_dimension,row_dimension]
+            col_dim_class =  Dimension.class_for_name(column_dimension)
+            col_dim_class.hierarchy_levels.each_key do |column_hierarchy_name|
+              row_dim_class = Dimension.class_for_name(row_dimension)
+              row_dim_class.hierarchy_levels.each_key do |row_hierarchy_name|
+                # Construct the aggregate meta data instance
+                meta_data_attributes = {
+                  :cube_name => self.name,
+                  :dimension1 => column_dimension.to_s,
+                  :dimension1_hierarchy => column_hierarchy_name.to_s,
+                  :dimension2 => row_dimension.to_s,
+                  :dimension2_hierarchy => row_hierarchy_name.to_s
+                }
+                conditions = []
+                condition_args = []
+                meta_data_attributes.each do |key, value|
+                  conditions << "#{key} = ?"
+                  condition_args << value
+                end
+                conditions = [conditions.join(' and ')] + condition_args
+                meta_data = AggregateMetaData.find(:first, :conditions => conditions)
+                unless meta_data
+                  meta_data = AggregateMetaData.create(meta_data_attributes)
+                end
+                
+                # Construct the aggregate class instance
+                aggregate_class = Class.new(ActiveWarehouse::Aggregate)
+                aggregate_class.name = "Agg#{meta_data.id}"
+                logger.debug "Constructed aggregate #{aggregate_class.name}"
+                aggregate_class.cube = self
+                aggregate_class.dimension1 = column_dimension
+                aggregate_class.dimension1_hierarchy_name = column_hierarchy_name
+                aggregate_class.dimension2 = row_dimension
+                aggregate_class.dimension2_hierarchy_name = row_hierarchy_name
+            
+                # Create the underlying aggregate storage table
+                # TODO: fix the bug of data not being found when a storage table rebuild occurs
+                force_storage_table_rebuild = options[:force] || aggregate_class.needs_rebuild?(last_modified)
+                logger.debug "Force storage table rebuild? #{force_storage_table_rebuild}"
+                aggregate_class.create_storage_table(force_storage_table_rebuild)
+            
+                # Keep a reference to the aggregate class instance
+                @aggregates[meta_data.id] = aggregate_class
+              end
+            end
+          end
+        end
+      end
+      
+    end
+    
+    public
+    def aggregate_map(column_dimension, column_hierarchy, row_dimension, row_hierarchy, cstage=0, rstage=0)
+      # Fill known cells
+      agg_map = AggregateMap.new
+      agg_records = aggregate_records(column_dimension, column_hierarchy, row_dimension, row_hierarchy)
+      agg_records.each do |agg_record|
+        # agg_record is an instance of Aggregate
+        # collect the aggregate record data fields into an array
+        data_array = agg_record.data_fields.collect{ |data_field_name| agg_record.send(data_field_name.to_sym) }
+        
+        # convert to an average where necessary
+        # TODO: implement
+
+        # add calculated fields to the data array
+        self.class.fact_class.calculated_fields.each do |calculated_field| 
+          options = self.class.fact_class.calculated_field_options[calculated_field]
+          data_array << options[:block].call(agg_record)
+        end
+        
+        # add the data array to the aggregate map
+        agg_map.add_data(agg_record.dimension2_path, agg_record.dimension1_path, data_array)
+      end
+      agg_map
+    end
+    
+    protected
+    # Return all of the Aggregate records for the specified dimensions and hierarchies
+    def aggregate_records(column_dimension, column_hierarchy, row_dimension, row_hierarchy)
+      k = Aggregate.key(column_dimension, column_hierarchy, row_dimension, row_hierarchy)
+      if aggregates[k].nil?
+        self.class.logger.debug("Aggregate #{k} not found in cache")
+        conditions = ['cube_name = ?', self.class.name]
+        conditions[0] << ' and dimension1 = ? and dimension1_hierarchy = ? and dimension2 = ? and dimension2_hierarchy = ?'
+        conditions << column_dimension.to_s
+        conditions << column_hierarchy.to_s
+        conditions << row_dimension.to_s
+        conditions << row_hierarchy.to_s
+        
+        conditions_reversed = ['cube_name = ?', self.class.name]
+        conditions_reversed[0] << ' and dimension1 = ? and dimension1_hierarchy = ? and dimension2 = ? and dimension2_hierarchy = ?'
+        conditions_reversed << row_dimension.to_s
+        conditions_reversed << row_hierarchy.to_s
+        conditions_reversed << column_dimension.to_s
+        conditions_reversed << column_hierarchy.to_s
+        
+        aggregate_meta_data = AggregateMetaData.find(:first, :conditions => conditions)
+        aggregate_meta_data ||= AggregateMetaData.find(:first, :conditions => conditions_reversed)
+        if aggregate_meta_data.nil?
+          self.class.rebuild
+          aggregate_meta_data = AggregateMetaData.find(:first, :conditions => conditions)
+          raise "Cannot find aggregate meta data for key #{k}" if aggregate_meta_data.nil?
+        end
+        aggregate_class = self.class.aggregates[aggregate_meta_data.id]
+        if aggregate_class.nil?
+          self.class.rebuild
+          aggregate_class = self.class.aggregates[aggregate_meta_data.id]
+          raise "Cannot find aggregate for id #{aggregate_meta_data.id}" if aggregate_class.nil?
+        end
+        #puts "Loading aggregate #{aggregate_meta_data.id}"
+        aggregates[k] = aggregate_class.find(:all)
+      end
+      aggregates[k]
+    end
+    
+    # Get a hash of all aggregate data
+    def aggregates
+      @aggregates ||= {}
+    end
+  end
+  
+  # In-memory map of aggregate values
+  class AggregateMap
+    attr_reader :length
+    
+    # Initialize the aggregate map
+    def initialize
+      @m = {}
+    end
+    
+    # Return true if the aggregate map includes the specified row path
+    def has_row_path?(row_path)
+      @m.has_key?(row_path)
+    end
+    
+    # Get the value for the specified row path, column path and field index
+    def value(row_path, col_path, field_index)
+      #puts "Getting value for #{row_path}, #{col_path} [field=#{field_index}]"
+      row = @m[row_path]
+      return 0 if row.nil?
+      col = row[col_path]
+      return 0 if col.nil?
+      return col[field_index] || 0
+    end
+    
+    # Get an array of the values for the specified row path and column path
+    def values(row_path, col_path)
+      row = @m[row_path]
+      return Array.new(length, 0) if row.nil?
+      col = row[col_path]
+      return Array.new(length, 0) if col.nil?
+      col
+    end
+    
+    # Add an array of data for the given row and column path
+    def add_data(row_path, col_path, data_array)
+      @length ||= data_array.length
+      #puts "Adding data for #{row_path}, #{col_path} [data=[#{data_array.join(',')}]]"
+      @m[row_path] ||= {}
+      @m[row_path][col_path] = data_array
+    end
+  end
+end