chicago-etl 0.0.9

data/spec/etl/load_dataset_builder_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+
+describe Chicago::ETL::LoadDatasetBuilder do
+  let(:db) { stub(:database).as_null_object }
+
+  before :each do
+    db.stub(:[]).with(:original_users).
+      and_return(TEST_DB[:original_users])
+    db.stub(:[]).with(:original_preferences).
+      and_return(TEST_DB[:original_preferences])
+    db[:original_users].stub(:columns).
+      and_return([:id, :name, :email])
+    db[:original_preferences].stub(:columns).
+      and_return([:id, :spam])
+  end
+
+  it "selects from the specified table" do
+    subject.table(:original_users)
+    subject.build(db, [:name]).opts[:from].should == [:original_users]
+  end
+
+  it "selects the columns from the table" do
+    subject.configure { table(:original_users) }
+
+    subject.build(db, [:id, :name]).opts[:select].should == [:id.qualify(:original_users), :name.qualify(:original_users)]
+  end
+
+  it "can handle column renaming" do
+    subject.configure do
+      table :original_users
+      provide :original_id, :id
+    end
+
+    subject.build(db, [:original_id, :name]).opts[:select].
+      should == [:id.qualify(:original_users).as(:original_id), :name.qualify(:original_users)]
+  end
+
+  it "can provide constructed columns" do
+    subject.configure do
+      table :original_users
+      provide :original_id, :foo.qualify(:bar)
+    end
+
+    subject.build(db, [:original_id, :name]).opts[:select].
+      should == [:foo.qualify(:bar).as(:original_id), :name.qualify(:original_users)]
+  end
+
+  it "left outer joins a denormalized table" do
+    subject.configure do
+      table :original_users
+      denormalize :original_preferences, :id => :id
+    end
+
+    subject.build(db, [:id, :name]).sql.should =~ /LEFT OUTER JOIN `original_preferences` ON \(`original_preferences`.`id` = `original_users`.`id`\)/
+  end
+
+  it "takes columns from the appropriate tables where possible" do
+    subject.configure do
+      table :original_users
+      denormalize :original_preferences, :id => :id
+    end
+
+    subject.build(db, [:id, :name, :spam]).opts[:select].
+      should == [:id.qualify(:original_users),
+                 :name.qualify(:original_users),
+                 :spam.qualify(:original_preferences)]
+  end
+
+  it "takes renames columns from denormalized tables" do
+    subject.configure do
+      table :original_users
+      denormalize :original_preferences, :id => :id
+      provide :email_allowed, :spam
+    end
+
+    subject.build(db, [:id, :name, :email_allowed]).opts[:select].
+      should include(:spam.qualify(:original_preferences).as(:email_allowed))
+  end
+
+  it "automatically renames ids of denormalized tables" do
+    subject.configure do
+      table :original_users
+      denormalize :original_preferences, :id => :id
+    end
+  end
+end

data/spec/etl/mysql_dumpfile_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe Chicago::ETL::MysqlDumpfile do
+  before :each do
+    @csv = mock(:csv)
+  end
+  
+  it "outputs specified column values in order" do
+    dumpfile = described_class.new(@csv, [:foo, :bar])
+    @csv.should_receive(:<<).with(["1", "2"])
+
+    dumpfile << {:foo => "1", :bar => "2", :baz => "not output"}
+  end
+
+  it "transforms values with a MysqlLoadFileValueTransformer" do
+    transformer = mock(:transformer)
+    Chicago::ETL::MysqlLoadFileValueTransformer.stub(:new).and_return(transformer)
+
+    transformer.should_receive(:transform).with("bar").and_return("baz")
+    @csv.should_receive(:<<).with(["baz"])
+
+    dumpfile = described_class.new(@csv, [:foo])
+    dumpfile << {:foo => "bar"}
+  end
+
+  it "will write a row only once with the same key" do
+    dumpfile = described_class.new(@csv, [:foo], :id)
+    @csv.should_receive(:<<).with(["bar"])
+    
+    dumpfile << {:id => 1, :foo => "bar"}
+    dumpfile << {:id => 1, :foo => "baz"}
+  end
+
+  it "will write a row multiple times if no key is specified" do
+    dumpfile = described_class.new(@csv, [:foo])
+    @csv.should_receive(:<<).with(["bar"])
+    @csv.should_receive(:<<).with(["baz"])
+    
+    dumpfile << {:id => 1, :foo => "bar"}
+    dumpfile << {:id => 1, :foo => "baz"}
+  end
+end

data/spec/etl/mysql_load_file_value_transformer_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Chicago::ETL::MysqlLoadFileValueTransformer do
+  it "transforms nil into \\N" do
+    subject.transform(nil).should == "\\N"
+  end
+
+  it "transforms true into '1'" do
+    subject.transform(true).should == "1"
+  end
+
+  it "transforms false into '0'" do
+    subject.transform(false).should == "0"
+  end
+
+  it "transforms times into mysql time format" do
+    subject.transform(Time.local(2011,01,02,10,30,50)).should == "2011-01-02 10:30:50"
+  end
+
+  it "transforms datetimes into mysql time format" do
+    subject.transform(DateTime.new(2011,01,02,10,30,50)).should == "2011-01-02 10:30:50"
+  end
+
+  it "transforms dates into mysql date format" do
+    subject.transform(Date.new(2011,01,02)).should == "2011-01-02"
+  end
+end

data/spec/etl/screens/composite_screen_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Screens::CompositeScreen do
+  let(:screen) do
+    i = 0
+
+    lambda {|row, errors| 
+      i += 1 
+      errors << i 
+      [row, errors]
+    }
+  end
+
+  it "calls all child screens" do
+    row, errors = described_class.new([screen, screen]).call({:a => 1}, [])
+    row.should == {:a => 1}
+    errors.should == [1,2]
+  end
+
+  it "supports variable arguments in the constructor" do
+    row, errors = described_class.new(screen, screen).call({:a => 1}, [])
+    row.should == {:a => 1}
+    errors.should == [1,2]
+  end
+end

data/spec/etl/screens/invalid_element_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Screens::InvalidElement do
+  let(:enum_col) { 
+    Chicago::Schema::Column.new(:enum, :string, :elements => ["Foo", "Unknown"], :default => "Unknown", :optional => true) 
+  }
+
+  it "has a severity of 3" do
+    described_class.new(:dimension_foo, enum_col).severity.should == 3
+  end
+
+  it "reports invalid element for enum columns" do
+    row, errors = described_class.new(:dimension_foo, enum_col).
+      call({:enum => "Bar"})
+    row.should == {:enum => 'Unknown'}
+
+    errors.first[:error].should == "Invalid Element"
+  end
+
+  it "does not report a valid element" do
+    row, errors = described_class.new(:dimension_foo, enum_col).
+      call({:enum => "foo"})
+    row.should == {:enum => 'foo'}
+
+    errors.should be_empty
+  end
+end

data/spec/etl/screens/missing_value_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Screens::MissingValue do
+  let(:string_col) { Chicago::Schema::Column.new(:str, :string) }
+  let(:int_col)    { Chicago::Schema::Column.new(:int, :integer) }
+  let(:bool_col)   { Chicago::Schema::Column.new(:bool, :boolean) }
+  
+  let(:descriptive_col) { 
+    Chicago::Schema::Column.new(:str, :string, :descriptive => true) 
+  }
+
+  let(:optional_col) { 
+    Chicago::Schema::Column.new(:str, :string, :optional => true) 
+  }
+
+  it "reports nil in an expected column as a missing value, with severity 2" do
+    row, errors = described_class.new(:dimension_foo, string_col).call({})
+    
+    errors.first[:table].should == "dimension_foo"
+    errors.first[:column].should == "str"
+    errors.first[:error].should == "Missing Value"
+    errors.first[:severity].should == 2
+  end
+
+  it "reports an empty string value in an expected column as a missing value" do
+    row, errors = described_class.new(:dimension_foo, string_col).
+      call({:str => "  "})
+    
+    errors.first[:error].should == "Missing Value"
+  end
+
+  it "does not report 0 as a missing value" do
+    row, errors = described_class.new(:dimension_foo, int_col).
+      call({:int => 0})
+    
+    errors.should be_empty
+  end
+
+  it "reports missing values with severity 1 if the column is descriptive" do
+    row, errors = described_class.new(:dimension_foo, descriptive_col).call({})
+    errors.first[:severity].should == 1
+  end
+
+  it "does not report boolean values as missing" do
+    row, errors = described_class.new(:dimension_foo, bool_col).call({})
+    errors.should be_empty
+  end
+
+  it "does not report optional columns as missing values" do
+    row, errors = described_class.new(:dimension_foo, optional_col).call({})
+    errors.should be_empty
+  end
+
+  it "fills in a default value for missing values" do
+    row, errors = described_class.new(:dimension_foo, optional_col).call({})
+    row.should == {:str => ''}
+  end
+end

data/spec/etl/screens/out_of_bounds_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Screens::OutOfBounds do
+  let(:int_col) { 
+    Chicago::Schema::Column.new(:int, :integer, :min => 0, :max => 100) 
+  }
+
+  let(:str_col) {
+    Chicago::Schema::Column.new(:str, :string, :min => 2, :max => 5) 
+  }
+
+  it "applies to numeric columns when the value is lower than the minimum" do
+    row, errors = described_class.new(:dimension_foo, int_col).
+      call(:int => -1)
+    
+    errors.first[:error].should == "Out Of Bounds"
+  end
+
+  it "applies to numeric columns when the value is above the minimum" do
+    row, errors = described_class.new(:dimension_foo, int_col).
+      call(:int => 101)
+    
+    errors.first[:error].should == "Out Of Bounds"
+  end
+
+  it "applies to string columns when the number of chars is below minimum" do
+    row, errors = described_class.new(:dimension_foo, str_col).
+      call(:str => "a")
+    
+    errors.first[:error].should == "Out Of Bounds"
+  end
+
+  it "applies to string columns when the number of chars is above maximum" do
+    row, errors = described_class.new(:dimension_foo, str_col).
+      call(:str => "abcdef")
+    
+    errors.first[:error].should == "Out Of Bounds"
+  end
+
+  it "does not apply to string values in range" do
+    row, errors = described_class.new(:dimension_foo, str_col).
+      call(:str => "abcde")
+    
+    errors.should be_empty
+  end
+
+  it "does not apply to numeric values in range" do
+    row, errors = described_class.new(:dimension_foo, int_col).
+      call(:int => 0)
+    
+    errors.should be_empty
+  end
+
+  it "has severity 2" do
+    described_class.new(:dimension_foo, int_col).severity.should == 2
+  end
+
+  it "does not replace values with default" do
+    row, errors = described_class.new(:dimension_foo, str_col).
+      call(:str => "a")
+
+    row.should == {:str => "a"}
+  end
+end

data/spec/etl/sequel/dependant_tables_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe Chicago::ETL::SequelExtensions::DependantTables do
+  it "returns the table in the from clause" do
+    TEST_DB[:foo].dependant_tables.should == [:foo]
+  end
+
+  it "returns tables from join clauses" do
+    TEST_DB[:foo].join(:bar).join(:baz).dependant_tables.
+      should == [:foo, :bar, :baz]
+  end
+
+  it "returns unique real tables from join clauses when aliased" do
+    TEST_DB[:foo].join(:bar).join(:bar.as(:baz)).dependant_tables.
+      should == [:foo, :bar]
+  end
+
+  it "returns real tables from 'from' clauses when aliased" do
+    TEST_DB[:foo.as(:bar)].join(:bar).join(:bar.as(:baz)).
+      dependant_tables.should == [:foo, :bar]
+  end
+
+  it "returns tables from nested datasets in the from clause" do
+    TEST_DB[TEST_DB[:foo].as(:bar)].dependant_tables.should == [:foo]
+  end
+
+  it "returns tables from nested datasets in the join clause" do
+    TEST_DB[:foo].join(TEST_DB[:bar].as(:baz)).dependant_tables.
+      should == [:foo, :bar]
+  end
+
+  it "handles unioned datasets" do
+    TEST_DB[:foo].union(TEST_DB[:bar]).union(TEST_DB[:baz]).
+      dependant_tables.should == [:foo, :bar, :baz]
+  end
+
+  it "handles unioned datasets where from_self is false" do
+    TEST_DB[:foo].union(TEST_DB[:bar], :from_self => false).
+      dependant_tables.should == [:foo, :bar]
+  end
+end

data/spec/etl/sequel/filter_to_etl_batch_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe Chicago::ETL::SequelExtensions::FilterToEtlBatch do
+  let(:batch) { stub(:batch, :id => 42) }
+
+  it "should do nothing to a table without an etl_batch_id column" do
+    TEST_DB.should_receive(:schema).with(:foo).and_return([])
+    TEST_DB[:foo].filter_to_etl_batch(batch).should == TEST_DB[:foo]
+  end
+
+  it "filters a table with an ETL batch id column" do
+    TEST_DB.should_receive(:schema).with(:foo).and_return([[:etl_batch_id, {}]])
+    TEST_DB[:foo].filter_to_etl_batch(batch).sql.
+      should include("\(`foo`.`etl_batch_id` = 42\)")
+  end
+
+  it "filters an aliased table with an ETL batch id column" do
+    TEST_DB.should_receive(:schema).with(:foo).and_return([[:etl_batch_id, {}]])
+    TEST_DB[:foo.as(:bar)].filter_to_etl_batch(batch).sql.
+      should include("\(`bar`.`etl_batch_id` = 42\)")
+  end
+
+  it "doesn't attempt to look for etl columns in nested queries" do
+    TEST_DB[TEST_DB[:foo].as(:bar)].filter_to_etl_batch(batch).sql.
+      should_not include("`bar`.`etl_batch_id` = 42")
+  end
+
+  it "filters based on joins" do
+    TEST_DB.should_receive(:schema).with(:baz).and_return([[:etl_batch_id, {}]])
+    TEST_DB.should_receive(:schema).with(:bar).and_return([])
+    TEST_DB.should_receive(:schema).with(:foo).and_return([])
+
+    sql = TEST_DB[:foo].join_table(:left_outer, :bar, :id => :id).join(:baz).filter_to_etl_batch(batch).sql
+    sql.should include("\(`baz`.`etl_batch_id` = 42\)")
+  end
+
+  it "filters based on joined aliases" do
+    TEST_DB.should_receive(:schema).with(:bar).and_return([[:etl_batch_id, {}]])
+    TEST_DB.should_receive(:schema).with(:foo).and_return([])
+
+    TEST_DB[:foo].join(:bar.as(:baz)).filter_to_etl_batch(batch).sql.
+      should include("\(`baz`.`etl_batch_id` = 42\)")
+  end
+
+  it "applies filters to each unioned dataset" do
+    TEST_DB.should_receive(:schema).with(:bar).and_return([[:etl_batch_id, {}]])
+    TEST_DB.should_receive(:schema).with(:foo).and_return([[:etl_batch_id, {}]])
+
+    sql = TEST_DB[:foo].union(TEST_DB[:bar], :from_self => false).filter_to_etl_batch(batch).sql
+
+    sql.should include("\(`foo`.`etl_batch_id` = 42\)")
+    sql.should include("\(`bar`.`etl_batch_id` = 42\)")
+  end
+end

data/spec/etl/sequel/load_data_infile_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe Chicago::ETL::SequelExtensions::LoadDataInfile do
+  before :each do
+    @sql = TEST_DB[:foo].load_csv_infile_sql("bar.csv", [:bar, :baz])
+  end
+
+  it "loads the data in the file" do
+    @sql.should include("LOAD DATA INFILE 'bar.csv'")
+  end
+
+  it "replaces rows currently in the table" do
+    @sql.should include("REPLACE INTO TABLE `foo`")
+  end
+
+  it "should be in the UTF 8 character set" do
+    @sql.should include("CHARACTER SET 'utf8'")
+  end
+
+  it "should escape with the \" character" do
+    @sql.should include("ESCAPED BY '\"'")
+  end
+
+  it "supports standard csv, with optional quoting" do
+    @sql.should include("FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'")
+  end
+
+  it "loads into the columns specified" do
+    @sql.should include("(`bar`,`baz`);")
+  end
+
+  it "can ignore instead of replacing rows" do
+    @sql = TEST_DB[:foo].insert_ignore.
+      load_csv_infile_sql("bar.csv", [:bar, :baz])
+    @sql.should include("IGNORE INTO TABLE `foo`")
+  end
+end