chicagowarehouse 0.3.1

data/lib/chicago/schema/column.rb

@@ -0,0 +1,221 @@
+require 'chicago/schema/named_element'
+
+module Chicago
+  module Schema
+    # A column in a dimension or fact record.
+    #
+    # The column definition is used to generate the options
+    # to create the column in the database schema, but also
+    # to provide an abstract definition of the column for views
+    # and other Data Warehouse code.
+    #
+    # You shouldn't need to create a Column manually - they
+    # are generally defined using the schema definition DSL.
+    #
+    # @api public
+    class Column
+      include Schema::NamedElement
+
+      # Creates a new column definition.
+      #
+      # name::  the name of the column.
+      # column_type::  the abstract type of the column. For example, :string.
+      #
+      # Options:
+      #
+      # min::      the minimum length/number of this column.
+      # max::      the maximum length/number of this column.
+      # range::    any object with a min & max method - overrides min/max (above).
+      # null::     whether this column can be null. False by default.
+      # elements:: the allowed values this column can take.
+      # default::  the default value for this column. 
+      # descriptive:: whether this column is purely descriptive and
+      # won't be used for grouping/filtering.
+      # internal:: the column is for internal use only, and shouldn't
+      #   be displayed/used directly in a user context
+      # optional:: the column isn't expected to be populated.
+      #
+      # @api private
+      def initialize(name, column_type, opts={})
+        @opts = normalize_opts(column_type, opts)
+        
+        super name, opts
+        
+        @column_type = column_type
+        if @opts[:countable].kind_of?(String)
+          @countable_label =  @opts[:countable]
+        end
+        @countable   = !! @opts[:countable]
+        @min         = @opts[:min]
+        @max         = @opts[:max]
+        @null        = @opts[:null]
+        @elements    = @opts[:elements]
+        @default     = @opts[:default]
+        @descriptive = !! @opts[:descriptive]
+        @internal    = !! @opts[:internal]
+        @optional    = !! (@opts.has_key?(:optional) ? @opts[:optional] : @opts[:null])
+      end
+
+      # Returns the type of this column. This is an abstract type,
+      # not a database type (for example :string, not :varchar).
+      attr_reader :column_type
+
+      # Returns the minimum value of this column, or nil.
+      attr_reader :min
+      
+      # Returns the minimum value of this column, or nil.
+      attr_reader :max
+      
+      # Returns an Array of allowed elements, or nil.
+      attr_reader :elements
+      
+      # Returns the explicit default as set in the database, or nil.
+      attr_reader :default
+
+      # Returns the calculated default value.
+      #
+      # This may be different from the explicit default - for example
+      # a boolean not-null column that has no explicit default will
+      # have a default of nil, and a default value of false.
+      #
+      # The distinction is important, otherwise values can end up
+      # missing in junk dimensions due to differences between what
+      # ruby and the database considers unique.
+      def default_value
+        if @default || null?
+          @default
+        elsif @column_type == :boolean
+          false
+        elsif numeric?
+          0
+        elsif textual?
+          ''
+        else
+          nil
+        end
+      end
+
+      attr_reader :countable_label
+
+      alias :key_name :name
+      
+      # Returns true if this column can be counted.
+      def countable?
+        @countable
+      end
+
+      # Returns true if this column should be indexed
+      def indexed?
+        ! descriptive?
+      end
+
+      # Returns true if this column is optional.
+      #
+      # Will be defaulted from whether the column allows null values,
+      # can be overridden (for dates etc).
+      def optional?
+        @optional
+      end
+      
+      # Returns true if this column should be ignored in user-facing
+      # parts of an application
+      def internal?
+        @internal
+      end
+
+      # Returns true if null values are allowed.
+      def null?
+        @null
+      end
+      
+      # Returns true if this column is just informational, and is not
+      # intended to be used as a filter.
+      def descriptive?
+        @descriptive
+      end
+
+      # Returns true if both definition's attributes are equal.
+      def ==(other)
+        other.kind_of?(self.class) && 
+          name == other.name && 
+          column_type == other.column_type && 
+          @opts == other.instance_variable_get(:@opts)
+      end
+
+      # Returns true if this column stores a numeric value.
+      def numeric?
+        @numeric ||= [:integer, :money, :percent, :decimal, :float].include?(column_type)
+      end
+
+      # Returns true if the column stores a textual value.
+      def textual?
+        @textual ||= [:string, :text].include?(column_type)
+      end
+
+      def hash #:nodoc:
+        name.hash
+      end
+
+      # Returns a hash of column options.
+      def to_hash
+        db_schema = {
+          :name => name,
+          :column_type => column_type,
+          :null => null?
+        }
+        db_schema[:default]  = default   if default || column_type == :timestamp
+        db_schema[:elements] = elements  if elements
+        db_schema[:size]     = size      if size
+        db_schema[:unsigned] = !! unsigned? if numeric?
+        db_schema
+      end
+
+      def qualify_by(table)
+        name.qualify(table)
+      end
+      
+      # Columns accept Visitors
+      def visit(visitor)
+        visitor.visit_column(self)
+      end
+      
+      private
+
+      def unsigned?
+        return @unsigned if defined? @unsigned      
+        default_unsigned = column_type == :percent || column_type == :money
+        @unsigned = min ? min >= 0 : default_unsigned
+      end
+
+      def size
+        @size ||= if @opts[:size]
+                    @opts[:size]
+                  elsif max && column_type == :string
+                    max
+                  elsif column_type == :money
+                    [12,2]
+                  elsif column_type == :percent
+                    [6,3]
+                  end
+      end
+      
+      def normalize_opts(type, opts)
+        opts = {:null => default_null(type), :min => default_min(type)}.merge(opts)
+        if opts[:range]
+          opts[:min] = opts[:range].min
+          opts[:max] = opts[:range].max
+          opts.delete(:range)
+        end
+        opts
+      end
+
+      def default_null(type)
+        [:date, :timestamp, :datetime].include?(type)
+      end
+
+      def default_min(type)
+        0 if type == :money
+      end
+    end
+  end
+end

data/lib/chicago/schema/column_parser.rb

@@ -0,0 +1,127 @@
+require 'chicago/schema/query_column'
+
+module Chicago
+  module Schema
+    # Parses AST column representations, returning an Array of
+    # QueryColumns.
+    #
+    # Columns can be simple dotted references, like
+    #
+    #     "sales.product.name"
+    #
+    # calculations like:
+    #
+    #     {:column => "sales.total", :op => "sum"}
+    #
+    # or pivoted calculations like:
+    #
+    #     {:column => "sales.total",
+    #      :op => "sum"
+    #      :pivot => "sales.date.year"}
+    #
+    class ColumnParser
+      # Creates a new ColumnParser for a schema.
+      def initialize(schema)
+        @schema = schema
+      end
+      
+      # Parses a column element. An element may be a string reference
+      # like "foo.bar", or more complicated like {:column =>
+      # "foo.bar", :op => "sum"}
+      #
+      # @return [Array<Column>] an array of columns. In most cases
+      #   this will be a 1-element array, unless the column is
+      #   pivoted.
+      def parse(elem)
+        [_parse(elem)].flatten
+      end
+
+      protected
+      
+      # Returns an Array of values, given a column to pivot with.
+      #
+      # May be overridden by subclasses.
+      #
+      # @raise UnimplementedError if a column with unknown or too many
+      #   elements is used as a pivot column. In future this
+      #   restriction may be lifted.
+      def pivotable_elements(pivot_col)
+        if pivot_col.column_type == :boolean
+          [true, false]
+        elsif pivot_col.elements
+          pivot_col.elements
+        elsif has_pivotable_integer_range?(pivot_col)
+          (pivot_col.min..pivot_col.max).to_a
+        else
+          raise UnimplementedError.new("General pivoting not yet support")
+        end
+      end
+
+      # Returns true if an Integer column can be used as pivot column.
+      #
+      # Default is to allow columns with a range 500 wide or less to
+      # be used as pivot columns.
+      #
+      # May be overridden by subclasses
+      #
+      # @return Boolean true if this column can be pivoted.
+      def has_pivotable_integer_range?(pivot_col)
+        pivot_col.column_type == :integer &&
+          pivot_col.max &&
+          pivot_col.min &&
+          (pivot_col.max - pivot_col.min <= 500)
+      end
+
+      private
+
+      def _parse(elem)
+        elem.kind_of?(Hash) ? complex_column(elem) : simple_column(elem)
+      end
+
+      def complex_column(elem)
+        elem.symbolize_keys!
+        elem[:pivot] ? pivoted_column(elem) : calculated_column(elem)
+      end
+
+      def pivoted_column(elem)
+        pivoted_column = _parse(elem[:column])
+        pivoted_by = _parse(elem[:pivot])
+        unit = [:avg, :count].include?(elem[:op].to_sym) ? nil : 0
+        pivoted_column.pivot(pivoted_by, pivotable_elements(pivoted_by), unit).map do |c|
+          c.calculate(elem[:op].to_sym)
+        end
+      end
+
+      def simple_column(elem)
+        table, col = parse_parts(elem)
+        QueryColumn.column(table, col, elem.to_sym)
+      end
+  
+      def calculated_column(elem)
+        col = _parse(elem[:column])
+        elem[:op] ? col.calculate(elem[:op].to_sym) : col
+      end
+
+      def parse_parts(str)
+        table, parts = parse_table(str)
+        col = table[parts.shift]
+        # To cope with bare dimension references.
+        col = table.original_key if col.nil?
+        
+        if col.kind_of?(Chicago::Schema::Dimension)
+          table = col
+          col = parts.empty? ? table : table[parts.first]
+        end
+
+        [table, col]
+      end
+
+      def parse_table(str)
+        parts = str.split('.').map(&:to_sym)
+        root = parts.shift
+        table = @schema.fact(root) || @schema.dimension(root)
+        [table, parts]
+      end
+    end
+  end
+end

data/lib/chicago/schema/dimension.rb

@@ -0,0 +1,129 @@
+require 'chicago/schema/table'
+
+module Chicago
+  module Schema
+    # The conventional format for dimension table names
+    DIMENSION_TABLE_FORMAT = "dimension_%s".freeze
+
+    # The conventional format for key table names.
+    KEY_TABLE_FORMAT = "keys_%s".freeze
+
+    # A dimension in the star schema.
+    #
+    # Dimensions contain denormalized values from various source
+    # systems, and are used to group and filter the fact tables. They
+    # may also be queried themselves.
+    #
+    # You shouldn't need to initialize a Dimension yourself - they
+    # should be created via StarSchema#define_dimension.
+    #
+    # @api public
+    class Dimension < Table
+      # Returns an array of Columns defined on this dimension.
+      #
+      # @see Chicago::Schema::Column.
+      attr_reader :columns
+
+      # @deprecated Use columns instead.
+      alias :column_definitions :columns
+      
+      # Returns all the human-friendly identifying columns for this
+      # dimension.
+      #
+      # There is no expectation that identifying values will be unique,
+      # but they are intended to identify a single record in a user
+      # friendly way.
+      attr_reader :identifiers
+
+      # The table used to generate/store dimension keys.
+      attr_reader :key_table_name
+
+      # Creates a new Dimension, named +name+.
+      #
+      # @param name the name of the dimension
+      # @option opts [Array] columns
+      # @option opts [Array] identifiers
+      # @option opts [Array] null_records an array of attribute
+      #   hashes, used to create null record rows in the database.
+      #   Hashes must have an :id key.
+      # @option opts [Array<Symbol>] natual_key an array of symbols,
+      #   representing a uniqueness constraint on the dimension.
+      # @option opts description a long text description about the dimension.
+      # @raise [Chicago::UnsafeNullRecordError] 
+      def initialize(name, opts={})
+        super
+        @columns = opts[:columns] || []
+        @identifiers = opts[:identifiers] || []
+        @null_records = opts[:null_records] || []
+        @table_name = sprintf(DIMENSION_TABLE_FORMAT, name).to_sym
+        @key_table_name = sprintf(KEY_TABLE_FORMAT, @table_name).to_sym
+        @predetermined_values = !! opts[:predetermined_values]
+        check_null_records
+      end
+
+      # Creates null records in a Database.
+      #
+      # This will overwrite any records that share the id with the
+      # null record, so be careful.
+      #
+      # Optionally provide an overridden table name, if you need to
+      # create null records for a temporary version of the table.
+      def create_null_records(db, overridden_table_name=nil)
+        table_to_populate = overridden_table_name || table_name
+        unless @null_records.empty?
+          db[table_to_populate].insert_replace.
+            insert_multiple(@null_records)
+          if db.table_exists?(key_table_name)
+            ids = @null_records.map {|r| {:dimension_id => r[:id]} }
+            db[key_table_name].insert_replace.insert_multiple(ids)
+          end
+        end
+      end
+
+      # Returns the main identifier for this record.
+      def main_identifier
+        @identifiers.first
+      end
+
+      # Returns true if this dimension can be identified as a concrete
+      # entity, with an original_id from a source system.
+      # 
+      # @todo change to be consistent with identifiers
+      def identifiable?
+        !! original_key
+      end
+
+      # Returns true if the set of values for this dimension is
+      # pretermined.
+      #
+      # Examples of this may be date dimensions, currency dimensions
+      # etc.
+      def has_predetermined_values?
+        @predetermined_values
+      end
+
+      # Returns the column that represents the id in the original
+      # source for the dimension.
+      #
+      # Currently this column *must* be called +original_id+
+      #
+      # @todo make configurable.
+      def original_key
+        @original_key ||= @columns.detect {|c| c.name == :original_id }
+      end
+
+      # Dimensions accept Visitors
+      def visit(visitor)
+        visitor.visit_dimension(self)
+      end
+      
+      private
+
+      def check_null_records
+        unless @null_records.all? {|h| h[:id] }
+          raise UnsafeNullRecordError.new "Null record defined without id field for dimension #{name}"
+        end
+      end
+    end
+  end
+end