chicagowarehouse 0.3.1

data/lib/chicago/star_schema.rb

@@ -0,0 +1,127 @@
+require 'chicago/errors'
+require 'chicago/schema/named_element_collection'
+require 'chicago/schema/column'
+require 'chicago/schema/measure'
+require 'chicago/schema/dimension_reference'
+require 'chicago/schema/dimension'
+require 'chicago/schema/fact'
+require 'chicago/schema/builders/fact_builder'
+require 'chicago/schema/builders/dimension_builder'
+require 'chicago/schema/builders/shrunken_dimension_builder'
+require 'chicago/schema/builders/column_builder'
+
+module Chicago
+  # A collection of facts & dimensions.
+  #
+  # @api public
+  class StarSchema
+    # A collection of all the facts defined in this schema.
+    # @return [Chicago::Schema::Fact]
+    attr_reader :facts
+
+    # a collection of all the dimensions defined in this schema.
+    # @return [Chicago::Schema::Dimension]
+    attr_reader :dimensions
+
+    # Creates a new star schema.
+    def initialize
+      @dimensions = Schema::NamedElementCollection.new
+      @facts = Schema::NamedElementCollection.new
+    end
+
+    # Returns a fact, named +name+
+    #
+    # @param [Symbol] name the name of the fact
+    # @return [Chicago::Schema::Fact]
+    def fact(name)
+      @facts[name]
+    end
+
+    # Returns a dimension, named +name+
+    #
+    # @param [Symbol] name the name of the dimension
+    # @return [Chicago::Schema::Dimension]
+    def dimension(name)
+      @dimensions[name]
+    end
+    
+    # Returns all dimensions and facts in this schema.
+    #
+    # @return [Array]
+    def tables
+      @dimensions.to_a + @facts.to_a
+    end
+    
+    # Adds a prebuilt schema table to the schema
+    #
+    # Schema tables may not be dupliates of already present tables in
+    # the schema.
+    #
+    # TODO: figure out how to deal with linked dimensions when adding
+    # facts.
+    def add(schema_table)
+      if schema_table.kind_of? Schema::Fact
+        collection = @facts
+      elsif schema_table.kind_of? Schema::Dimension
+        collection = @dimensions
+      end
+      
+      add_to_collection collection, schema_table
+    end
+    
+    # Defines a fact table named +name+ in this schema.
+    #
+    # @see Chicago::Schema::Builders::FactBuilder
+    # @return [Chicago::Schema::Fact] the defined fact.
+    # @raise Chicago::MissingDefinitionError
+    def define_fact(name, &block)
+      add Schema::Builders::FactBuilder.new(self).build(name, &block)
+    end
+
+    # Defines a dimension table named +name+ in this schema.
+    #
+    # For example:
+    #
+    #    @schema.define_dimension(:date) do
+    #      columns do
+    #        date   :date
+    #        year   :year
+    #        string :month
+    #        ...
+    #      end
+    #
+    #      natural_key :date
+    #      null_record :id => 1, :month => "Unknown Month"
+    #    end
+    #
+    # @see Chicago::Schema::Builders::DimensionBuilder
+    # @return [Chicago::Schema::Dimension] the defined dimension.
+    def define_dimension(name, &block)
+      add Schema::Builders::DimensionBuilder.new(self).build(name, &block)
+    end
+
+    # Defines a shrunken dimension table named +name+ in this schema.
+    #
+    # +base_name+ is the name of the base dimension that the shrunken
+    # dimension is derived from; this base dimention must already be
+    # defined.
+    #
+    # @see Chicago::Schema::Builders::ShrunkenDimensionBuilder
+    # @raise [Chicago::MissingDefinitionError] if the base dimension is not defined.
+    # @return [Chicago::Schema::Dimension] the defined dimension.
+    def define_shrunken_dimension(name, base_name, &block)
+      add Schema::Builders::ShrunkenDimensionBuilder.new(self, base_name).
+        build(name, &block)
+    end
+
+    private
+
+    def add_to_collection(collection, schema_table)
+      if collection.contain?(schema_table)
+        raise DuplicateTableError.new("#{schema_table.class} '#{schema_table.name}' has already been defined.")
+      end
+
+      collection << schema_table
+    end
+  end
+end

data/spec/core_ext/sequel_extensions_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe Symbol do
+  it "should have a distinct method" do
+    :foo.distinct.should be_kind_of(Sequel::SQL::DistinctExpression)
+  end
+end
+
+describe String do
+  it "should have a distinct method" do
+    "foo".distinct.should be_kind_of(Sequel::SQL::DistinctExpression)
+  end
+end
+
+describe Sequel::SQL::Expression do
+  it "should have a distinct method" do
+    :if.sql_function(true, 1, 0).distinct.should be_kind_of(Sequel::SQL::DistinctExpression)
+  end
+end
+
+describe Sequel::SQL::DistinctExpression do
+  it "has a reader for the wrapped expression" do
+    described_class.new(:foo).expression.should == :foo
+  end
+
+  it "is rendered as 'DISTINCT expression'" do
+    described_class.new(:foo).to_s(TEST_DB[:foo]).should == "DISTINCT `foo`"
+  end
+end

data/spec/data/month_spec.rb

@@ -0,0 +1,67 @@
+require "spec_helper"
+
+describe Chicago::Data::Month do
+
+  [:january, :february, :march, :april, :may, :june, :july, :august, :september, :october, :november, :december].each do |month|
+    it "should return a Month for #{month}" do
+      Chicago::Data::Month.send(month).should be_kind_of(Chicago::Data::Month)
+    end
+  end
+
+  it "should have a name" do
+    Chicago::Data::Month.january.name.should == "January"
+  end
+
+  it "should return the name from #to_s" do
+    Chicago::Data::Month.january.to_s.should == "January"
+  end
+
+  it "should have a short name" do
+    Chicago::Data::Month.january.short_name.should == "Jan"
+  end
+
+  it "should have a number" do
+    Chicago::Data::Month.january.to_i.should == 1
+  end
+
+  it "should return a date in a year" do
+    Chicago::Data::Month.january.in(2009).should == Date.new(2009,1,1)
+  end
+
+  it "should be the same instance on mutliple calls to a month name" do
+    Chicago::Data::Month.january.object_id.should == Chicago::Data::Month.january.object_id
+  end
+
+  it "should be comparable" do
+    Chicago::Data::Month.january.should < Chicago::Data::Month.february
+    Chicago::Data::Month.december.should > Chicago::Data::Month.january
+  end
+
+  it "should not be constructable" do
+    lambda { Chicago::Data::Month("Foo", 13) }.should raise_error
+  end
+
+  it "should be parsable from a short string" do
+    Chicago::Data::Month.parse("Jan").should == Chicago::Data::Month.january
+  end
+
+  it "should be parsable from a short string uncaptialized" do
+    Chicago::Data::Month.parse("jan").should == Chicago::Data::Month.january
+  end
+
+  it "should be parsable from a full name" do
+    Chicago::Data::Month.parse("December").should == Chicago::Data::Month.december
+  end
+
+  it "should be parsable from a full name uncaptialized" do
+    Chicago::Data::Month.parse("january").should == Chicago::Data::Month.january
+  end
+
+  it "should be parsable from an integer" do
+    Chicago::Data::Month.parse(1).should == Chicago::Data::Month.january
+  end
+
+  it "should return nil from a string that isn't a month name" do
+    Chicago::Data::Month.parse("maybe").should == nil
+  end
+end

data/spec/database/db_type_converter_spec.rb

@@ -0,0 +1,125 @@
+require "spec_helper"
+
+shared_examples_for "All DB type converters" do
+  context "#db_type" do
+    before(:each) do
+      @dimension = stub(:dimension)
+    end
+    
+    it "should return :varchar for a string column" do
+      column = Schema::Column.new(:id, :string)
+      @tc.db_type(column).should == :varchar
+    end
+
+    it "should return :char for a string column that has equal max and min attributes" do
+      column = Schema::Column.new(:id, :string, :min => 2, :max => 2)
+      @tc.db_type(column).should == :char
+    end
+
+    it "should return :integer for an column with a max but no min attribute set" do
+      column = Schema::Column.new(:id, :integer, :max => 127)
+      @tc.db_type(column).should == :integer
+    end
+
+    it "should return :decimal for a money column type" do
+      column = Schema::Column.new(:id, :money)
+      @tc.db_type(column).should == :decimal
+    end
+
+    it "should return :decimal for a percent column type" do
+      column = Schema::Column.new(:id, :percent)
+      @tc.db_type(column).should == :decimal
+    end
+
+    it "should assume any other type is a database type and return it" do
+      column = Schema::Column.new(:id, :foo)
+      @tc.db_type(column).should == :foo
+    end
+  end
+end
+
+describe "DbTypeConverter.for_db" do
+  before :each do
+    @mock_db = mock()
+  end
+
+  it "should return a type converter specific to MySQL if the database type is :mysql" do
+    @mock_db.should_receive(:database_type).and_return(:mysql)
+
+    converter = Database::TypeConverters::DbTypeConverter.for_db(@mock_db)
+    converter.should be_kind_of(Database::TypeConverters::MysqlTypeConverter)
+  end
+
+  it "should return a generic type converter for an unknown database type" do
+    @mock_db.should_receive(:database_type).and_return(:foodb)
+
+    converter = Database::TypeConverters::DbTypeConverter.for_db(@mock_db)
+    converter.should be_kind_of(Database::TypeConverters::DbTypeConverter)
+  end
+end
+
+describe "Generic DbTypeConverter" do
+  it_behaves_like "All DB type converters"
+
+  before :each do 
+    @tc = Database::TypeConverters::DbTypeConverter.new
+  end
+
+  { :smallint  => [-32768, 32767],
+    :smallint  => [0, 65535],
+  }.each do |expected_db_type, range|
+
+    it "should create a #{expected_db_type} if the maximum column value < #{range.max} and min is >= #{range.min}" do
+      column = Schema::Column.new(:id, :integer, :max => range.max, :min => range.min)
+      @tc.db_type(column).should == expected_db_type
+    end
+  end
+end
+
+describe Chicago::Database::TypeConverters::MysqlTypeConverter do
+  it_behaves_like "All DB type converters"
+
+  before :each do
+    @tc = Database::TypeConverters::MysqlTypeConverter.new
+  end
+
+  context "#db_type" do
+    { :tinyint   => [-127, 128],
+      :tinyint   => [0, 255],
+      :smallint  => [-32768, 32767],
+      :smallint  => [0, 65535],
+      :mediumint => [-8388608, 8388607],
+      :mediumint => [0, 16777215],
+      :integer   => [-2147483648, 2147483647],
+      :integer   => [0, 4294967295],
+      :bigint    => [-9223372036854775808, 9223372036854775807],
+      :bigint    => [0, 18446744073709551615]
+
+    }.each do |expected_db_type, range|
+      
+      it "should return #{expected_db_type} if the maximum column value < #{range.max} and min is >= #{range.min}" do
+        column = Schema::Column.new(:id, :integer, :max => range.max, :min => range.min)
+        @tc.db_type(column).should == expected_db_type
+      end
+    end
+
+    it "should raise an ArgumentError if either of the min/max values are out of bounds" do
+      column = Schema::Column.new(:id, 
+                                    :integer, 
+                                    :min => 0, 
+                                    :max => 18_446_744_073_709_551_616)
+      
+      lambda { @tc.db_type(column) }.should raise_error(ArgumentError)
+    end
+
+    it "should return :enum if the column definition has elements" do
+      column = Schema::Column.new(:id, :string, :elements => ["A", "B"])
+      @tc.db_type(column).should == :enum
+    end
+
+    it "should return :varchar if the column definition has a large number of elements" do
+      column = Schema::Column.new(:id, :string, :elements => stub(:size => 70_000))
+      @tc.db_type(column).should == :varchar
+    end
+  end
+end

data/spec/database/migration_file_writer_spec.rb

@@ -0,0 +1,37 @@
+require "spec_helper"
+
+describe Chicago::Database::MigrationFileWriter do
+  before :each do
+    @mock_db = mock(:db)
+    @mock_db.stub(:database_type).and_return(:generic)
+    @builder = described_class.new(@mock_db, "schema")
+  end
+
+  it "should return a migration file name using the timestamp of now" do
+    now = Time.local(2010, 1, 1, 12, 30)
+    Time.should_receive(:now).and_return(now)
+    @builder.migration_file.should == "schema/20100101123000_auto_migration.rb"
+  end
+
+  it "should write out a migration file generated by Sequel::MigrationBuilder" do
+    schema = stub(:schema)
+
+    generator = stub(:generator)
+    generator.should_receive(:traverse).
+      with(schema).and_return(:table => :definitions)
+    Chicago::Database::SchemaGenerator.stub(:new).and_return(generator)
+    
+    migration_builder = mock(:migration_builder)
+    migration_builder.should_receive(:generate_migration).
+      with(:table => :definitions).and_return("migration content")
+    Sequel::MigrationBuilder.stub(:new).and_return(migration_builder)
+    
+    file = StringIO.new
+    File.stub(:open).and_yield(file)
+    
+    @builder.write_migration_file(schema)
+
+    file.rewind
+    file.read.should == "migration content"
+  end
+end

data/spec/database/schema_generator_spec.rb

@@ -0,0 +1,199 @@
+require 'spec_helper'
+require 'chicago/database/schema_generator'
+
+describe Chicago::Database::SchemaGenerator do
+  subject { described_class.new(Chicago::Database::TypeConverters::DbTypeConverter.new) }
+  
+  it_behaves_like "a schema visitor"
+  
+  describe "#visit_fact" do
+    before :all do
+      schema = Chicago::StarSchema.new
+      schema.define_dimension :customer
+      schema.define_dimension :product      
+      @fact = schema.define_fact(:sales) do
+        dimensions :customer, :product
+        degenerate_dimensions { string :reference }
+        measures { integer :quantity }
+        natural_key :customer, :reference
+      end
+    end
+    
+    it "should define a sales_facts table" do
+      subject.visit_fact(@fact).should have_key(:facts_sales)
+    end
+
+    it "should have an unsigned integer :id column" do
+      expected = {:name => :id, :column_type => :integer, :unsigned => true}
+      subject.visit_fact(@fact)[:facts_sales][:columns].
+        should include(expected)
+    end
+
+    it "should define :id as the primary key" do
+      subject.visit_fact(@fact)[:facts_sales][:primary_key].should == [:id]
+    end
+
+    it "should include a hash of table options" do
+      subject.visit_fact(@fact)[:facts_sales][:table_options].should == {}
+    end
+
+    it "should have a table type of MyISAM for mysql" do
+      subject.type_converter = Chicago::Database::TypeConverters::MysqlTypeConverter.new
+      subject.visit_fact(@fact)[:facts_sales][:table_options].should == {:engine => "myisam"}
+    end
+
+    it "should output the dimension foreign key columns" do
+      [{:name => :customer_dimension_id, :column_type => :integer, :unsigned => true, :null => false},
+       {:name => :product_dimension_id, :column_type => :integer, :unsigned => true, :null => false}
+      ].each do |column|
+        subject.visit_fact(@fact)[:facts_sales][:columns].should include(column)
+      end
+    end
+
+    it "should output the degenerate dimension columns" do
+      subject.visit_fact(@fact)[:facts_sales][:columns].should include({:name => :reference, :column_type => :varchar, :null => false})
+    end
+
+    it "should output the measure columns" do
+      subject.visit_fact(@fact)[:facts_sales][:columns].should include({:name => :quantity, :column_type => :integer, :null => true, :unsigned => false})
+    end
+
+    it "should define non-unique indexes for every dimension" do
+      subject.visit_fact(@fact)[:facts_sales][:indexes].should == {
+        :product_idx => { :columns => :product_dimension_id, :unique => false },
+        :customer_idx => { :columns => [:customer_dimension_id, :reference], :unique => true },
+        :reference_idx => { :columns => :reference, :unique => false }
+      }
+    end
+  end
+
+  describe "#visit_dimension" do
+    before :each do
+      @schema = Chicago::StarSchema.new
+    end
+
+    it "should define a user_dimension table" do
+      @dimension = @schema.define_dimension(:user)
+      subject.visit_dimension(@dimension).keys.should include(:dimension_user)
+    end
+
+    it "should have an unsigned integer :id column" do
+      @dimension = @schema.define_dimension(:user)
+      expected = {:name => :id, :column_type => :integer, :unsigned => true}
+      subject.visit_dimension(@dimension)[:dimension_user][:columns].should include(expected)
+    end
+
+    it "should define :id as the primary key" do
+      @dimension = @schema.define_dimension(:user)
+      subject.visit_dimension(@dimension)[:dimension_user][:primary_key].should == [:id]
+    end
+
+    it "should include a hash of table options" do
+      @dimension = @schema.define_dimension(:user)
+      subject.visit_dimension(@dimension)[:dimension_user][:table_options].should == {}
+    end
+
+    it "should have a table type of MyISAM for mysql" do
+      @dimension = @schema.define_dimension(:user)
+      subject.type_converter = Chicago::Database::TypeConverters::MysqlTypeConverter.new
+      subject.visit_dimension(@dimension)[:dimension_user][:table_options].should == {:engine => "myisam"}
+    end
+
+    it "should include the sequel schema for the defined columns" do
+      @dimension = @schema.define_dimension(:user) do
+        columns do
+          string :username, :max => 10
+        end
+      end
+
+      expected = {:name => :username, :column_type => :varchar, :size => 10, :null => false}
+      subject.visit_dimension(@dimension)[:dimension_user][:columns].should include(expected)
+    end
+
+    it "should output indexes for every column that isn't descriptive" do
+      @dimension = @schema.define_dimension(:user) do
+        columns do
+          string :foo, :descriptive => true
+          string :bar
+          string :baz
+        end
+      end
+
+      expected = {
+        :bar_idx => {:columns => :bar, :unique => false},
+        :baz_idx => {:columns => :baz, :unique => false}
+      }
+      subject.visit_dimension(@dimension)[:dimension_user][:indexes].should == expected
+    end
+
+    it "should output a natural_key unique index for the natural key" do
+      @dimension = @schema.define_dimension(:user) do
+        columns do
+          string :foo, :descriptive => true
+          string :bar
+          string :baz
+        end
+        natural_key :bar, :baz
+      end
+
+      expected = {
+        :bar_idx => {:columns => [:bar, :baz], :unique => true}, 
+        :baz_idx => {:columns => :baz, :unique => false}
+      }
+      subject.visit_dimension(@dimension)[:dimension_user][:indexes].should == expected
+    end
+
+    # This just supports internal convention at the moment
+    it "should create a key mapping table if an original_id column is present" do
+      @dimension = @schema.define_dimension(:user) do
+        columns do
+          integer :original_id, :min => 0
+          string :username, :max => 10
+        end
+      end
+
+      key_table = subject.visit_dimension(@dimension)[:keys_dimension_user]
+      key_table.should_not be_nil
+      key_table[:primary_key].should == [:original_id]
+
+      expected = [{:name => :original_id, :column_type => :integer, :null => false, :unsigned => true},
+                  {:name => :dimension_id, :column_type => :integer, :null => false, :unsigned => true}]
+      key_table[:columns].should == expected
+    end
+
+    it "creates a mapping table with a binary column, for dimensions with no original_id" do
+      @dimension = @schema.define_dimension(:user) do
+        columns do
+          string :username, :max => 10
+          natural_key :username
+        end
+      end
+
+      key_table = subject.visit_dimension(@dimension)[:keys_dimension_user]
+      key_table.should_not be_nil
+      key_table[:primary_key].should == [:original_id]
+
+      expected = [{:name => :original_id, :column_type => :binary, :null => false, :size => 16},
+                  {:name => :dimension_id, :column_type => :integer, :null => false, :unsigned => true}]
+      key_table[:columns].should == expected
+    end
+
+    it "doesn't create a key table for static dimensions" do
+      @dimension = @schema.define_dimension(:currency) do
+        has_predetermined_values
+
+        columns do
+          string :currency
+        end
+      end
+
+      subject.visit_dimension(@dimension)[:keys_dimension_currency].should be_nil
+    end
+
+    it "should have an unsigned integer :etl_batch_id column" do
+      @dimension = @schema.define_dimension(:user)
+      expected = {:name => :etl_batch_id, :column_type => :integer, :unsigned => true}
+      subject.visit_dimension(@dimension)[:dimension_user][:columns].should include(expected)
+    end
+  end
+end