chicago-etl 0.0.13 → 0.1.0

data/spec/etl/dataset_batch_stage_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe Chicago::ETL::DatasetBatchStage do
+  let(:pipeline_stage) { mock(:pipeline_stage).as_null_object }
+  let(:dataset) { mock(:dataset).as_null_object }
+  let(:stage) { described_class.new(:foo, dataset, pipeline_stage) }
+  let(:etl_batch) { stub(:etl_batch) }
+
+  it "has a name" do
+    stage.name.should == :foo
+  end
+
+  it "should set the inserted at time on the default sink" do
+    sink = Chicago::Flow::ArraySink.new(:foo)
+    pipeline_stage.stub(:sink).with(:default).and_return(sink)
+    stage.pipeline_stage.should == pipeline_stage
+
+    sink.constant_values[:_inserted_at].should_not be_nil
+  end
+
+  it "filters the dataset to the batch" do
+    dataset.should_recieve(:filter_to_etl_batch).with(etl_batch)
+    stage.source(etl_batch)
+  end
+
+  it "does not filter the dataset if re-extracting" do
+    dataset.should_not_recieve(:filter_to_etl_batch)
+    stage.source(etl_batch, true)
+  end
+
+  it "can filter via a custom strategy" do
+    dataset.should_not_recieve(:filter_to_etl_batch)
+
+    filter_strategy = lambda {|ds, batch| ds }
+    described_class.new(:foo, dataset, pipeline_stage, :filter_strategy => filter_strategy).
+      source(etl_batch)
+  end
+
+  it "executes the pipeline stage using a DatasetSource" do
+    pipeline_stage.should_receive(:execute).
+      with(kind_of(Chicago::Flow::DatasetSource))
+    stage.execute(etl_batch, true)
+  end
+
+  it "truncates any sinks if truncate_pre_load has been set" do
+    stage = described_class.new(:foo, dataset, pipeline_stage,
+                                :truncate_pre_load => true)
+
+    sink = Chicago::Flow::ArraySink.new(:output)
+    sink << {:foo => "foo"}
+    pipeline_stage.stub(:sinks).and_return([sink])
+    stage.execute(etl_batch)
+    sink.data.should == []
+  end
+end

data/spec/etl/key_builder_spec.rb

@@ -39,8 +39,9 @@ describe Chicago::ETL::KeyBuilder do
   before :each do
     @db = stub(:staging_database).as_null_object
     @db.stub(:[]).and_return(stub(:max => nil, :select_hash => {}))
-    @writer = stub(:writer).as_null_object
-    Chicago::ETL::BufferingInsertWriter.stub(:new).and_return(@writer)
+    @sink = stub(:sink).as_null_object
+    Chicago::ETL::SchemaTableSinkFactory.stub(:new).
+      and_return(stub(:factory, :key_sink => @sink))
   end
 
   describe "for identifiable dimensions" do
@@ -50,20 +51,20 @@ describe Chicago::ETL::KeyBuilder do
 
     it "returns an incrementing key, given a row" do
       builder = described_class.for_table(@dimension, @db)
-      builder.key(:original_id => 2).should == 1
-      builder.key(:original_id => 3).should == 2
+      builder.key(:original_id => 2).first.should == 1
+      builder.key(:original_id => 3).first.should == 2
     end
 
     it "returns the same key for the same record" do
       builder = described_class.for_table(@dimension, @db)
-      builder.key(:original_id => 2).should == 1
-      builder.key(:original_id => 2).should == 1
+      builder.key(:original_id => 2).first.should == 1
+      builder.key(:original_id => 2).first.should == 1
     end
 
     it "takes into account the current maximum key in the database" do
       @db.stub(:[]).with(:keys_dimension_user).and_return(stub(:max => 2, :select_hash => {}))
       builder = described_class.for_table(@dimension, @db)
-      builder.key(:original_id => 1).should == 3
+      builder.key(:original_id => 1).first.should == 3
     end
 
     it "returns previously created keys" do
@@ -71,55 +72,14 @@ describe Chicago::ETL::KeyBuilder do
       @db.stub(:[]).with(:keys_dimension_user).and_return(dataset)
 
       builder = described_class.for_table(@dimension, @db)
-      builder.key(:original_id => 30).should == 2
-      builder.key(:original_id => 40).should == 1
+      builder.key(:original_id => 30).first.should == 2
+      builder.key(:original_id => 40).first.should == 1
     end
 
     it "raises an error when original_id isn't present in the row" do
       builder = described_class.for_table(@dimension, @db)
       expect { builder.key(:foo => :bar) }.to raise_error(Chicago::ETL::KeyError)
     end
-
-    it "flushes new keys to a key table" do
-      pending
-      dataset = stub(:dataset, :max => 1, :select_hash => {40 => 1})
-      dataset.stub(:insert_replace => dataset)
-      @db.stub(:[]).with(:keys_dimension_user).and_return(dataset)
-
-      dataset.should_receive(:multi_insert).
-        with([{:original_id => 30, :dimension_id => 2}])
-                                              
-      builder = described_class.for_table(@dimension, @db)
-      builder.key(:original_id => 30)
-      builder.key(:original_id => 40)
-      builder.flush
-    end
-
-    it "flushes new keys only once" do
-      pending
-      dataset = stub(:dataset, :max => 1, :select_hash => {40 => 1})
-      dataset.stub(:insert_replace => dataset)
-      @db.stub(:[]).with(:keys_dimension_user).and_return(dataset)
-
-      dataset.should_receive(:multi_insert).
-        with([{:original_id => 30, :dimension_id => 2}])
-      dataset.should_receive(:multi_insert).with([])
-                                              
-      builder = described_class.for_table(@dimension, @db)
-      builder.key(:original_id => 30)
-      builder.key(:original_id => 40)
-      builder.flush
-      builder.flush
-    end
-
-    it "replaces old mappings with new values" do
-      pending
-      dataset = stub(:dataset, :max => 1, :select_hash => {40 => 1}, :multi_insert => nil)
-      @db.stub(:[]).with(:keys_dimension_user).and_return(dataset)
-
-      dataset.should_receive(:insert_replace).and_return(dataset)
-      described_class.for_table(@dimension, @db).flush
-    end
   end
 
   describe "for non-identifiable dimensions with an existing hash" do
@@ -129,25 +89,18 @@ describe Chicago::ETL::KeyBuilder do
     end
 
     it "returns an incrementing key, given a row" do
-      @builder.key(:hash => "aaa").should == 1
-      @builder.key(:hash => "aab").should == 2
+      @builder.key(:hash => "aaa").first.should == 1
+      @builder.key(:hash => "aab").first.should == 2
     end
 
     it "returns the same incrementing key" do
-      @builder.key(:hash => "aaa").should == 1
-      @builder.key(:hash => "aaa").should == 1
+      @builder.key(:hash => "aaa").first.should == 1
+      @builder.key(:hash => "aaa").first.should == 1
     end
 
     it "returns the same incrementing key, ignoring case" do
-      @builder.key(:hash => "aaa").should == 1
-      @builder.key(:hash => "AAA").should == 1
-    end
-
-    it "inserts the hash as a binary literal" do
-      # Yuck. Don't like the implementation test, but mock
-      # expectations fail here for some reason, maybe because of the
-      # Sequel::LiteralString?
-      @builder.key_for_insert(@builder.original_key(:hash => "aaa")).should == "0xAAA".lit
+      @builder.key(:hash => "aaa").first.should == 1
+      @builder.key(:hash => "AAA").first.should == 1
     end
   end
 
@@ -158,32 +111,25 @@ describe Chicago::ETL::KeyBuilder do
 
     it "returns an incrementing key, given a row" do
       @builder.key(:line1 => "some street", :post_code => "TW3 X45").
-        should == 1
+        first.should == 1
       @builder.key(:line1 => "some road", :post_code => "TW3 X45").
-        should == 2
+        first.should == 2
     end
 
     it "returns the same incrementing key, ignoring case" do
       @builder.key(:line1 => "some street", :post_code => "TW3 X45").
-        should == 1
+        first.should == 1
       @builder.key(:line1 => "some STREET", :post_code => "TW3 X45").
-        should == 1
+        first.should == 1
     end
 
     it "can override default hash preparation" do
       @builder.hash_preparation = lambda {|c| c }
 
       @builder.key(:line1 => "some street", :post_code => "TW3 X45").
-        should == 1
+        first.should == 1
       @builder.key(:line1 => "some STREET", :post_code => "TW3 X45").
-        should == 2
-    end
-
-    it "inserts the hash as a binary literal" do
-      # Yuck. Don't like the implementation test, but mock
-      # expectations fail here for some reason, maybe because of the
-      # Sequel::LiteralString?
-      @builder.key_for_insert(@builder.original_key(:line1 => "some street", :post_code => "TW3 X45")).should == "0x817860F2417EB83D81FEA9D82E6B213A".lit
+        first.should == 2
     end
 
     it "selects the Hex version of the binary column for the cache" do
@@ -210,18 +156,14 @@ describe Chicago::ETL::KeyBuilder do
     end
 
     it "increments the id, regardless of row equality" do
-      @builder.key({}).should == 1
-      @builder.key({}).should == 2
+      @builder.key({}).first.should == 1
+      @builder.key({}).first.should == 2
     end
 
     it "increments from the last id stored id in the fact table" do
       @db.stub(:[]).with(:facts_addresses).and_return(stub(:max => 100, :select_hash => {}))
       @builder = described_class.for_table(@schema.fact(:addresses), @db)
-      @builder.key({}).should == 101
-    end
-
-    it "supports the flush interface as a no-op" do
-      lambda { @builder.flush }.should_not raise_error
+      @builder.key({}).first.should == 101
     end
   end
 end

data/spec/etl/pipeline_stage_builder_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe Chicago::ETL::LoadPipelineStageBuilder do
+  let(:dimension) { stub(:dimension).as_null_object }
+  let(:db) { stub(:db).as_null_object }
+  let(:sink_factory) { stub(:sink_factory).as_null_object }
+
+  before(:each) {
+    Chicago::ETL::SchemaTableSinkFactory.stub(:new).and_return(sink_factory)
+  }
+
+  it "should exclude columns from the sink" do
+    sink_factory.should_receive(:sink).
+      with(:ignore => false, :exclude => [:foo]).
+      and_return(stub(:sink).as_null_object)
+
+    described_class.new(db, dimension).build do
+      load_separately :foo
+    end
+  end
+
+  it "can specify rows are not going to be replaced" do
+    sink_factory.should_receive(:sink).
+      with(:ignore => true, :exclude => []).
+      and_return(stub(:sink).as_null_object)
+
+    described_class.new(db, dimension).build do
+      ignore_present_rows
+    end
+  end
+
+  it "can add key mappings" do
+    stage = described_class.new(db, dimension).build do
+      key_mapping :bar, :original_id
+    end
+
+    stage.sink(:bar).should_not be_nil
+  end
+end

data/spec/etl/schema_table_sink_factory_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+
+describe Chicago::ETL::SchemaTableSinkFactory do
+  let(:db) { stub(:db) }
+
+  let(:dimension) {
+    Chicago::Schema::Builders::DimensionBuilder.new(stub(:schema)).build(:foo) do
+      columns do
+        string :bar
+        integer :baz
+      end
+    end
+  }
+
+  let(:sink_class) { Chicago::Flow::MysqlFileSink }
+
+  it "builds a MysqlFileSink" do
+    sink_class.should_receive(:new).
+      with(db, :dimension_foo, [:id, :bar, :baz], {})
+    
+    described_class.new(db, dimension).sink
+  end
+
+  it "allows rows to be ignored instead of replaced" do
+    sink_class.should_receive(:new).
+      with(db, :dimension_foo, [:id, :bar, :baz], {:ignore => true})
+
+    described_class.new(db, dimension).sink(:ignore => true)
+  end
+
+  it "allows an explicit filepath to be specified" do
+    sink_class.should_receive(:new).
+      with(db, :dimension_foo, [:id, :bar, :baz], {:filepath => "foo"})
+
+    described_class.new(db, dimension).sink(:filepath => "foo")
+  end
+
+  it "can exclude columns from a dimension" do
+    sink_class.should_receive(:new).
+      with(db, :dimension_foo, [:id, :bar], {})
+
+    described_class.new(db, dimension).sink(:exclude => :baz)
+  end
+
+  it "builds the key table sink" do
+    sink = stub(:sink).as_null_object
+    sink_class.should_receive(:new).
+      with(db, :keys_dimension_foo, [:original_id, :dimension_id], {}).
+      and_return(sink)
+
+    described_class.new(db, dimension).key_sink()
+  end
+
+  it "builds other explicit key table sinks" do
+    sink = stub(:sink).as_null_object
+    sink_class.should_receive(:new).
+      with(db, :keys_foo, [:original_id, :dimension_id], {}).
+      and_return(sink)
+    
+    described_class.new(db, dimension).key_sink(:table => :keys_foo)
+  end
+
+  it "builds an error sink" do
+    sink_class.should_receive(:new).
+      with(db, :etl_error_log, [:column, :row_id, :error, :severity, :error_detail], {}).and_return(stub.as_null_object)
+
+    described_class.new(db, dimension).error_sink
+  end
+end

data/spec/etl/screens/invalid_element_spec.rb

@@ -5,23 +5,22 @@ describe Chicago::ETL::Screens::InvalidElement do
     Chicago::Schema::Column.new(:enum, :string, :elements => ["Foo", "Unknown"], :default => "Unknown", :optional => true) 
   }
 
+  let(:transformation) {
+    described_class.new(:column => enum_col)
+  }
+
   it "has a severity of 3" do
-    described_class.new(:dimension_foo, enum_col).severity.should == 3
+    transformation.severity.should == 3
   end
 
   it "reports invalid element for enum columns" do
-    row, errors = described_class.new(:dimension_foo, enum_col).
-      call({:enum => "Bar"})
-    row.should == {:enum => 'Unknown'}
-
-    errors.first[:error].should == "Invalid Element"
+    row = transformation.process_row({:enum => "Bar"})
+    
+    row[:enum].should == 'Unknown'
+    row[:_errors].first[:error].should == "Invalid Element"
   end
 
   it "does not report a valid element" do
-    row, errors = described_class.new(:dimension_foo, enum_col).
-      call({:enum => "foo"})
-    row.should == {:enum => 'foo'}
-
-    errors.should be_empty
+    transformation.process_row({:enum => "foo"}).should == {:enum => 'foo'}
   end
 end

data/spec/etl/screens/missing_value_spec.rb

@@ -14,45 +14,45 @@ describe Chicago::ETL::Screens::MissingValue do
   }
 
   it "reports nil in an expected column as a missing value, with severity 2" do
-    row, errors = described_class.new(:dimension_foo, string_col).call({})
-    
-    errors.first[:table].should == "dimension_foo"
-    errors.first[:column].should == "str"
-    errors.first[:error].should == "Missing Value"
-    errors.first[:severity].should == 2
+    row = described_class.new(:column => string_col).process_row({})
+                              
+    error = row[:_errors].first
+    error[:column].should == "str"
+    error[:error].should == "Missing Value"
+    error[:severity].should == 2
   end
 
   it "reports an empty string value in an expected column as a missing value" do
-    row, errors = described_class.new(:dimension_foo, string_col).
-      call({:str => "  "})
+    row = described_class.
+      new(:column => string_col).
+      process_row({:str => "  "})
     
-    errors.first[:error].should == "Missing Value"
+    row[:_errors].should_not be_nil
   end
 
   it "does not report 0 as a missing value" do
-    row, errors = described_class.new(:dimension_foo, int_col).
-      call({:int => 0})
-    
-    errors.should be_empty
+    row = described_class.new(:column => int_col).
+      process_row({:int => 0})
+    row[:_errors].should be_nil
   end
 
   it "reports missing values with severity 1 if the column is descriptive" do
-    row, errors = described_class.new(:dimension_foo, descriptive_col).call({})
-    errors.first[:severity].should == 1
+    row = described_class.new(:column => descriptive_col).process_row({})
+    row[:_errors].last[:severity].should == 1
   end
 
   it "does not report boolean values as missing" do
-    row, errors = described_class.new(:dimension_foo, bool_col).call({})
-    errors.should be_empty
+    row = described_class.new(:column => bool_col).process_row({})
+    row[:_errors].should be_nil
   end
 
   it "does not report optional columns as missing values" do
-    row, errors = described_class.new(:dimension_foo, optional_col).call({})
-    errors.should be_empty
+    row = described_class.new(:column => optional_col).process_row({})
+    row[:_errors].should be_nil
   end
 
   it "fills in a default value for missing values" do
-    row, errors = described_class.new(:dimension_foo, optional_col).call({})
-    row.should == {:str => ''}
+    row = described_class.new(:column => optional_col).process_row({})
+    row[:str].should == ''
   end
 end

data/spec/etl/screens/out_of_bounds_spec.rb

@@ -9,56 +9,48 @@ describe Chicago::ETL::Screens::OutOfBounds do
     Chicago::Schema::Column.new(:str, :string, :min => 2, :max => 5) 
   }
 
+  let(:int_transformation) {
+    described_class.new(:column => int_col)
+  }
+
+  let(:str_transformation) {
+    described_class.new(:column => str_col)
+  }
+
   it "applies to numeric columns when the value is lower than the minimum" do
-    row, errors = described_class.new(:dimension_foo, int_col).
-      call(:int => -1)
-    
-    errors.first[:error].should == "Out Of Bounds"
+    row = int_transformation.process_row(:int => -1)
+    row[:_errors].first[:error].should == "Out Of Bounds"
   end
 
   it "applies to numeric columns when the value is above the minimum" do
-    row, errors = described_class.new(:dimension_foo, int_col).
-      call(:int => 101)
-    
-    errors.first[:error].should == "Out Of Bounds"
+    row = int_transformation.process_row(:int => 101)
+    row[:_errors].first[:error].should == "Out Of Bounds"
   end
 
   it "applies to string columns when the number of chars is below minimum" do
-    row, errors = described_class.new(:dimension_foo, str_col).
-      call(:str => "a")
-    
-    errors.first[:error].should == "Out Of Bounds"
+    row = str_transformation.process_row(:str => "a")
+    row[:_errors].first[:error].should == "Out Of Bounds"
   end
 
   it "applies to string columns when the number of chars is above maximum" do
-    row, errors = described_class.new(:dimension_foo, str_col).
-      call(:str => "abcdef")
-    
-    errors.first[:error].should == "Out Of Bounds"
+    row = str_transformation.process_row(:str => "abcdef")
+    row[:_errors].first[:error].should == "Out Of Bounds"
   end
 
   it "does not apply to string values in range" do
-    row, errors = described_class.new(:dimension_foo, str_col).
-      call(:str => "abcde")
-    
-    errors.should be_empty
+    str_transformation.process_row(:str => "abcde").
+      should_not have_key(:_errors)
   end
 
   it "does not apply to numeric values in range" do
-    row, errors = described_class.new(:dimension_foo, int_col).
-      call(:int => 0)
-    
-    errors.should be_empty
+    int_transformation.process_row(:int => 0).should_not have_key(:_errors)
   end
 
   it "has severity 2" do
-    described_class.new(:dimension_foo, int_col).severity.should == 2
+    int_transformation.severity.should == 2
   end
 
   it "does not replace values with default" do
-    row, errors = described_class.new(:dimension_foo, str_col).
-      call(:str => "a")
-
-    row.should == {:str => "a"}
+    str_transformation.process_row(:str => "a")[:str].should == "a"
   end
 end

data/spec/etl/transformations_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Transformations::DemultiplexErrors do
+  it "declares it adds things to the error stream" do
+    subject.output_streams.should include(:error)
+  end
+
+  it "does nothing to a row without an :_errors key" do
+    subject.process({}).should == [{}]
+  end
+
+  it "removes the :_error key from the row" do
+    subject.process(:_errors => [{:error => 1}]).first.should == {}
+  end
+
+  it "adds the errors onto the error stream" do
+    subject.process(:_errors => [{:error => 1}]).last.should == {
+      :error => 1,
+      Chicago::Flow::STREAM => :error
+    }
+  end
+end
+
+describe Chicago::ETL::Transformations::WrittenRowFilter do
+  it "only lets the first row through" do
+    filter = described_class.new(:key => :id)
+    filter.process(:id => 1).should == {:id => 1}
+    filter.process(:id => 2).should == {:id => 2}
+    filter.process(:id => 1).should be_nil
+  end
+
+  it "requires a key option" do
+    described_class.required_options.should include(:key)
+  end
+end
+
+describe Chicago::ETL::Transformations::AddKey do
+  let(:key_builder) { stub(:key_builder, :key => 42) }
+  let(:transform) { described_class.new(:key_builder => key_builder) }
+
+  it "requires a key builder" do
+    described_class.required_options.should include(:key_builder)
+  end
+
+  it "adds the key to the row" do
+    transform.process({}).should == {:id => 42}
+  end
+
+  it "adds the key to any rows in an embedded :_errors key" do
+    transform.process({:_errors => [{}]}).
+      should == {:id => 42, :_errors => [{:row_id => 42}]}
+  end
+
+  it "should declare that it adds the :id field" do
+    transform.added_fields.should == [:id]
+  end
+
+  it "should declare that it writes to the dimension_key stream" do
+    transform.output_streams.should include(:dimension_key)
+  end
+
+  it "should return a new row on the dimension_key stream" do
+    key_builder.stub(:key => [42, {:original_id => 42}])
+    transform.process({}).last.should == {:_stream => :dimension_key, :original_id => 42}
+  end
+end
+
+describe Chicago::ETL::Transformations::DimensionKeyMapping do
+  let(:transform) {
+    described_class.new(:original_key => :original_id,
+                        :key_table => :keys_foo)
+  }
+  
+  it "should require an original_key and a key table" do
+    described_class.required_options.should == [:original_key, :key_table]
+  end
+
+  it "removes the key from the stream" do
+    transform.process({:original_id => 1}).first.should == {}
+    transform.removed_fields.should == [:original_id]
+  end
+
+  it "links the original key with the id on the stream" do
+    transform.process({:original_id => 1, :id => 2}).last.
+      should == {:_stream => :keys_foo, :original_id => 1, :dimension_id => 2}
+  end
+end
+
+describe Chicago::ETL::Transformations::HashColumns do
+  it "requires a columns option" do
+    described_class.required_options.should include(:columns)
+  end
+
+  it "adds a hash field to the row" do
+    Digest::MD5.stub(:hexdigest).with("ab").and_return("a")
+
+    transform = described_class.new(:columns => [:a, :b])
+    transform.added_fields.should == [:hash]
+    transform.process(:a => 'a', :b => 'b')[:hash].should == "A"
+  end
+
+  it "can add the hash to an arbitrary output field" do
+    Digest::MD5.stub(:hexdigest).with("ab").and_return("a")
+    transform = described_class.new(:columns => [:a, :b], 
+                                    :output_field => :foo)
+    transform.added_fields.should == [:foo]
+    transform.process(:a => 'a', :b => 'b')[:foo].should == "A"
+  end
+end

data/spec/flow/array_sink_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Chicago::Flow::ArraySink do
+  let(:sink) { described_class.new(:foo) }
+
+  it "has a name" do
+    sink.name.should == :foo
+  end
+
+  it "stores rows in #data" do
+    sink << {:a => 1}
+    sink.data.should == [{:a => 1}]
+  end
+
+  it "merges constant values into the sink row" do
+    sink.set_constant_values(:number => 1).should == sink
+    sink << {:a => 1}
+    sink.data.should == [{:a => 1, :number => 1}]
+  end
+
+  it "can be truncated" do
+    sink << {:a => 1}
+    sink.truncate
+    sink.data.should be_empty
+  end
+end