ncs_mdes_warehouse 0.6.2 → 0.7.0

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.6.2'
+    VERSION = '0.7.0'
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib", "generated_models"]
 
-  s.add_dependency 'ncs_mdes', '~> 0.6', '>= 0.6.1'
+  s.add_dependency 'ncs_mdes', '~> 0.7'
   s.add_dependency 'ncs_navigator_configuration', '~> 0.2'
 
   # Post-3.2.4 breaks DataMapper due to https://github.com/rails/rails/pull/6857
@@ -40,6 +40,8 @@ Gem::Specification.new do |s|
   s.add_dependency 'bcdatabase', '~> 1.1'
   s.add_dependency 'actionmailer', '~> 3.0'
 
+  s.add_dependency 'treetop'
+
   s.add_development_dependency 'rspec', '~> 2.6'
   s.add_development_dependency 'rake', '~> 0.9.2'
   s.add_development_dependency 'yard', '~> 0.7.2'

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -41,6 +41,19 @@ module NcsNavigator::Warehouse
       end
     end
 
+    describe '#foreign_key_index' do
+      it 'provides an index by default' do
+        config.foreign_key_index.should respond_to(:record_and_verify)
+      end
+
+      it 'allows the index to be overridden' do
+        i = Transformers::ForeignKeyIndex.new
+
+        config.foreign_key_index = i
+        config.foreign_key_index.should be(i)
+      end
+    end
+
     describe 'add_post_etl_hook' do
       let(:success_hook) {
         Object.new.tap do |o|

data/spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb

@@ -13,14 +13,39 @@ module NcsNavigator::Warehouse::Transformers
       property :name, String, :required => true
     end
 
+    class Subsample
+      include ::DataMapper::Resource
+
+      property :id, Integer, :key => true
+      property :name, String
+      property :age, Integer
+      belongs_to :sample,
+        'NcsNavigator::Warehouse::Transformers::Sample', :child_key => [ :sample_id ], :required => false
+    end
+
     before(:all) do
       DataMapper.finalize
     end
 
+    shared_examples 'a foreign key index updater' do
+      it 'records the IDs of saved records as seen' do
+        expected_saved_record_ids.each do |id|
+          foreign_key_index.seen?(Sample, id).should be_true
+        end
+      end
+
+      it 'does not record the IDs of not-saved records as seen' do
+        expected_not_saved_record_ids.each do |id|
+          foreign_key_index.seen?(Sample, id).should be_false
+        end
+      end
+    end
+
     describe '#transform' do
       let(:config) {
         NcsNavigator::Warehouse::Configuration.new.tap do |c|
           c.log_file = tmpdir + 'enum_transformer_test.log'
+          c.foreign_key_index = foreign_key_index
         end
       }
       let(:records) { [
@@ -29,6 +54,7 @@ module NcsNavigator::Warehouse::Transformers
           Sample.new(:id => 3, :psu_id => '20000030', :name => 'Three')
         ] }
       let(:transform_status) { NcsNavigator::Warehouse::TransformStatus.memory_only('test') }
+      let(:foreign_key_index) { ForeignKeyIndex.new(:existing_key_provider => nil) }
       subject { EnumTransformer.new(config, records) }
 
       it 'saves valid records when all are valid' do
@@ -82,6 +108,11 @@ module NcsNavigator::Warehouse::Transformers
         it 'saves the other instances' do
           # in `before`
         end
+
+        let(:expected_saved_record_ids) { [1, 2] }
+        let(:expected_not_saved_record_ids) { [3] }
+
+        include_examples 'a foreign key index updater'
       end
 
       describe 'with an instance with an invalid PSU' do
@@ -91,28 +122,25 @@ module NcsNavigator::Warehouse::Transformers
           records[2].should_receive(:save).and_return(true)
 
           records[1].psu_id = '20000041'
+
+          subject.transform(transform_status)
         end
 
         it 'does not save that instance' do
-          subject.transform(transform_status)
+          # in before
         end
 
         it 'saves the other instances' do
-          subject.transform(transform_status)
+          # in before
         end
 
         it 'records an error' do
-          subject.transform(transform_status)
           transform_status.transform_errors.collect(&:record_id).should == ['2']
         end
 
         describe 'the recorded error' do
           let(:error) { transform_status.transform_errors.first }
 
-          before do
-            subject.transform(transform_status)
-          end
-
           it 'has the correct model class' do
             error.model_class.should == Sample.to_s
           end
@@ -134,6 +162,30 @@ module NcsNavigator::Warehouse::Transformers
             error.attribute_value.should == '"20000041"'
           end
         end
+
+        let(:expected_saved_record_ids) { [1, 3] }
+        let(:expected_not_saved_record_ids) { [2] }
+
+        include_examples 'a foreign key index updater'
+      end
+
+      describe 'with an unsatisfied foreign key' do
+        let(:error) { transform_status.transform_errors.first }
+
+        before do
+          records << Subsample.new(:id => 3, :sample_id => 912)
+
+          records.each do |m|
+            m.stub!(:valid?).and_return(true)
+            m.stub!(:save).and_return(true)
+          end
+
+          subject.transform(transform_status)
+        end
+
+        it 'reports the unsatisfied foreign key' do
+          error.message.should =~ /Unsatisfied foreign key sample_id=912/
+        end
       end
 
       describe 'with an unsaveable instance' do
@@ -155,6 +207,11 @@ module NcsNavigator::Warehouse::Transformers
         it 'saves the saveable instances' do
           # in `before`
         end
+
+        let(:expected_saved_record_ids) { [1, 3] }
+        let(:expected_not_saved_record_ids) { [2] }
+
+        include_examples 'a foreign key index updater'
       end
 
       describe 'with an instance that errors on save' do
@@ -177,6 +234,11 @@ module NcsNavigator::Warehouse::Transformers
         it 'saves the saveable instances' do
           # in `before`
         end
+
+        let(:expected_saved_record_ids) { [2, 3] }
+        let(:expected_not_saved_record_ids) { [1] }
+
+        include_examples 'a foreign key index updater'
       end
 
       describe 'with an enumeration that throws an exception' do
@@ -262,8 +324,14 @@ module NcsNavigator::Warehouse::Transformers
           EnumTransformer.new(config, records, :filters => filters)
         end
 
-        it 'applies all the filters' do
+        before do
+          records.each do |r|
+            r.stub!(:valid?).and_return(true)
+            r.stub!(:save).and_return(true)
+          end
+        end
 
+        it 'applies all the filters' do
           transformer_with_filters(filter_one, filter_two).transform(transform_status)
 
           records.each { |m| m.name.should == 'filtered' }
@@ -272,10 +340,93 @@ module NcsNavigator::Warehouse::Transformers
         it 'saves only the results from the filter' do
           records[0].should_not_receive(:save)
           records[2].should_not_receive(:save)
-          records[1].should_receive(:save).exactly(3).times
+          records[1].should_receive(:save).exactly(3).times.and_return(true)
 
           transformer_with_filters(replacing_filter).transform(transform_status)
         end
+
+        context do
+          before do
+            transformer_with_filters(replacing_filter).transform(transform_status)
+          end
+
+          let(:expected_saved_record_ids) { [2] }
+          let(:expected_not_saved_record_ids) { [1, 3] }
+
+          include_examples 'a foreign key index updater'
+        end
+      end
+
+      describe 'duplicates modes' do
+        let(:sample_1_en) { Subsample.new(:id => 1, :name => 'One', :age => 80) }
+        let(:sample_1_es) { Subsample.new(:id => 1, :name => 'Uno', :age => nil) }
+
+        let(:records) { [sample_1_en, sample_1_es] }
+
+        def transformer_for_dups(mode)
+          EnumTransformer.new(config, records, :duplicates => mode)
+        end
+
+        it 'defaults the duplicates mode to :error' do
+          EnumTransformer.new(config, records).duplicates.should == :error
+        end
+
+        it 'accepts a duplicates mode option' do
+          transformer_for_dups(:ignore).duplicates.should == :ignore
+        end
+
+        it 'fails for an unknown duplicates mode' do
+          lambda { transformer_for_dups(:explode) }.
+            should raise_error(/Unknown duplicates mode :explode\./)
+        end
+
+        describe ':error' do
+          it 'tries to save everything' do
+            sample_1_en.should_receive(:save).and_return(true)
+            sample_1_es.should_receive(:save).and_return(true)
+
+            # If we weren't mocking, the transaction commit would result in
+            # an exception. I don't think it's important to test that here.
+            transformer_for_dups(:error).transform(transform_status)
+            transform_status.transform_errors.should == []
+          end
+        end
+
+        describe ':ignore' do
+          it 'does not try to save the duplicate' do
+            sample_1_en.should_receive(:save).and_return(true)
+            sample_1_es.should_not_receive(:save)
+
+            transformer_for_dups(:ignore).transform(transform_status)
+            transform_status.transform_errors.should == []
+          end
+        end
+
+        describe ':replace' do
+          let(:reloaded_sample) { sample_1_en.clone }
+
+          before do
+            sample_1_en.should_receive(:save).once.and_return(true)
+            sample_1_es.should_not_receive(:save)
+
+            Subsample.should_receive(:get).with(1).and_return(reloaded_sample)
+            reloaded_sample.should_receive(:save).once.and_return(true)
+
+            transformer_for_dups(:replace).transform(transform_status)
+          end
+
+          it 'loads and updates the existing record' do
+            transform_status.transform_errors.should == []
+          end
+
+          it 'replaces a set property with a new value' do
+            reloaded_sample.name.should == 'Uno'
+          end
+
+          it 'replaces a nil property with nil' do
+            reloaded_sample.age.should be_nil
+          end
+        end
       end
     end
   end

data/spec/ncs_navigator/warehouse/transformers/foreign_key_index/database_key_provider_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+class NcsNavigator::Warehouse::Transformers::ForeignKeyIndex
+  describe DatabaseKeyProvider, :slow, :use_mdes, :use_database, :modifies_warehouse_state do
+    let(:provider) { DatabaseKeyProvider.new }
+    let(:psu_model) { mdes_model(:Psu) }
+
+    before do
+      DataMapper.repository(:mdes_warehouse_working) do
+        psu_model.new(:sc_id => '20000013', :psu_id => '20000054', :recruit_type => '-4').save.should be_true
+      end
+    end
+
+    it 'provides the existing IDs for a model out of the working repo' do
+      provider.existing_keys(psu_model).should == %w(20000054)
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/foreign_key_index_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers
+  describe ForeignKeyIndex, :modifies_warehouse_state do
+    class Addr
+      include ::DataMapper::Resource
+
+      property :psu_id, String
+      property :id, Integer, :key => true
+
+      belongs_to :old_one, self, :child_key => [ :old_one_id ]
+      belongs_to :frob, 'NcsNavigator::Warehouse::Transformers::Frob', :child_key => [ :frob_id ]
+    end
+
+    class Frob
+      include ::DataMapper::Resource
+
+      property :psu_id, String
+      property :id, Integer, :key => true
+    end
+
+    before(:each) do
+      DataMapper.finalize
+    end
+
+    let(:fk_index) { ForeignKeyIndex.new(:existing_key_provider => key_provider) }
+    let(:key_provider) { ForeignKeyIndex::StaticKeyProvider.new(Addr.to_s => [120, 180]) }
+    let(:transform_status) { NcsNavigator::Warehouse::TransformStatus.memory_only('test') }
+    let(:errors) { transform_status.transform_errors }
+
+    describe '#initialize' do
+      describe ':existing_key_provider' do
+        it 'defaults to a database provider' do
+          ForeignKeyIndex.new.existing_key_provider.should be_a ForeignKeyIndex::DatabaseKeyProvider
+        end
+
+        it 'uses the provided instance' do
+          fk_index.existing_key_provider.should be key_provider
+        end
+
+        it 'disables existing key resolution when set to nil explicitly' do
+          provider = ForeignKeyIndex.new(:existing_key_provider => nil).existing_key_provider
+          provider.existing_keys(Addr).should be_nil
+        end
+      end
+    end
+
+    describe 'reporting errors' do
+      before do
+        fk_index.record_and_verify(Frob.new(:id => 8))
+        fk_index.record_and_verify(Frob.new(:id => 16))
+      end
+
+      it 'does not report for a key which is satisfied when it is first recorded' do
+        fk_index.record_and_verify(Addr.new(:frob_id => 8, :id => 1))
+        fk_index.report_errors(transform_status)
+
+        errors.should == []
+      end
+
+      it 'does not report for a key which is not initially satisfied but is later' do
+        fk_index.record_and_verify(Addr.new(:frob_id => 4, :id => 1))
+        fk_index.record_and_verify(Frob.new(:id => 4))
+        fk_index.report_errors(transform_status)
+
+        errors.should == []
+      end
+
+      it 'does not report for a key which is provided by the external key provider' do
+        fk_index.record_and_verify(Addr.new(:id => 8, :old_one_id => 120))
+        fk_index.report_errors(transform_status)
+
+        errors.should == []
+      end
+
+      describe 'when a key is never satisfied' do
+        before do
+          fk_index.record_and_verify(Addr.new(:frob_id => 4, :id => 1))
+          fk_index.report_errors(transform_status)
+        end
+
+        it 'reports the error' do
+          errors.size.should == 1
+        end
+
+        it 'includes the correct model_class' do
+          errors.first.model_class.should == Addr.to_s
+        end
+
+        it 'includes the correct record_id' do
+          errors.first.record_id.should == '1'
+        end
+
+        it 'includes a useful message' do
+          errors.first.message.should ==
+            'Unsatisfied foreign key frob_id=4 referencing NcsNavigator::Warehouse::Transformers::Frob.'
+        end
+
+        it 'only reports the error once if #report_errors is called multiple times' do
+          fk_index.report_errors(transform_status)
+
+          errors.size.should == 1
+        end
+      end
+
+      it 'reports multiple failed references for a single record' do
+        fk_index.record_and_verify(Addr.new(:id => 1, :frob_id => 2, :old_one_id => 3))
+        fk_index.report_errors(transform_status)
+
+        errors.collect { |e| e.message.match(/foreign key (\S+)/)[1] }.sort.should ==
+          %w(frob_id=2 old_one_id=3)
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/sql_transformer_spec.rb

@@ -0,0 +1,157 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers
+  describe SqlTransformer do
+    describe '.extract_statements' do
+      it 'splits on semicolons' do
+        SqlTransformer.extract_statements("SELECT 1; SELECT 2; SELECT 'A';").should == [
+          'SELECT 1',
+          'SELECT 2',
+          "SELECT 'A'"
+        ]
+      end
+
+      it 'includes the last statement even if it does not end with a semicolon' do
+        SqlTransformer.extract_statements("SELECT 3; SELECT 7; SELECT 'B'").should == [
+          'SELECT 3',
+          'SELECT 7',
+          "SELECT 'B'"
+        ]
+      end
+
+      it 'ignores semicolons inside of string literals' do
+        SqlTransformer.extract_statements("SELECT ';'; SELECT 7;").should == [
+          "SELECT ';'",
+          'SELECT 7'
+        ]
+      end
+
+      it 'handles strings with escaped quotes properly' do
+        SqlTransformer.extract_statements("SELECT 'Fred'';s'; SELECT '7';").should == [
+          "SELECT 'Fred'';s'",
+          "SELECT '7'"
+        ]
+      end
+
+      it 'handles E-strings with escaped quotes properly'
+    end
+
+    let(:options) { { :adapter => mock('adapter') } }
+    let(:transformer) { SqlTransformer.new(spec_config, options) }
+
+    describe '#initialize' do
+      it 'accepts an array of statements' do
+        options[:statements] = ['SELECT 1', 'SELECT 8']
+
+        transformer.statements.should == ['SELECT 1', 'SELECT 8']
+      end
+
+      it 'accepts and splits a string containing a script' do
+        options[:script] = "SELECT 1; SELECT ';'"
+
+        transformer.statements.should == ['SELECT 1', "SELECT ';'"]
+      end
+
+      it 'can read a script from a file' do
+        filename = tmpdir + 'a_script.sql'
+
+        filename.open('w') do |f|
+          f.puts "SELECT 1;"
+          f.puts "SELECT ';';"
+        end
+
+        options[:file] = filename.to_s
+
+        transformer.statements.should == ['SELECT 1', "SELECT ';'"]
+      end
+
+      it 'accepts a name' do
+        options[:statements] = []
+        options[:name] = 'Foo'
+        transformer.name.should == 'Foo'
+      end
+
+      it 'provides a default name' do
+        options[:statements] = []
+        transformer.name.should == 'SQL Transformer with 0 statements'
+      end
+    end
+
+    describe '#transform', :use_database do
+      let(:adapter) { mock('adapter') }
+
+      let(:statements) {
+        [
+          'UPDATE a SET x=42',
+          'UPDATE b SET y=42',
+          'UPDATE c SET z=42'
+        ]
+      }
+
+      let(:transformer) {
+        SqlTransformer.new(spec_config, :statements => statements, :adapter => adapter)
+      }
+
+      let(:transform_status) {
+        NcsNavigator::Warehouse::TransformStatus.memory_only('test')
+      }
+
+      def result(count)
+        mock('result').tap do |m|
+          m.stub!(:affected_rows).and_return(count)
+        end
+      end
+
+      describe 'when successful' do
+        before do
+          adapter.should_receive(:execute).with(statements[0]).ordered.and_return(result(4))
+          adapter.should_receive(:execute).with(statements[1]).ordered.and_return(result(7))
+          adapter.should_receive(:execute).with(statements[2]).ordered.and_return(result(1))
+
+          transformer.transform(transform_status)
+        end
+
+        it 'executes all statements in series' do
+          # in before
+        end
+
+        it 'stores the total affected record count' do
+          transform_status.record_count.should == (4 + 7 + 1)
+        end
+      end
+
+      describe 'when there is an error' do
+        before do
+          adapter.should_receive(:execute).with(statements[0]).ordered.and_return(result(4))
+          adapter.should_receive(:execute).with(statements[1]).ordered.and_raise('Give up now')
+          adapter.should_not_receive(:execute).with(statements[2])
+
+          transformer.transform(transform_status)
+        end
+
+        it 'stops on the first statement with an error' do
+          # in before
+        end
+
+        it 'records a transform error' do
+          transform_status.transform_errors.size.should == 1
+        end
+
+        describe 'the error' do
+          let(:the_error) {
+            transform_status.transform_errors.first
+          }
+
+          it 'has an appropriate message' do
+            the_error.message.should =~
+              /^Exception while executing SQL statement "UPDATE b SET y=42" \(2 of 3\).\s+RuntimeError: Give up now/
+          end
+        end
+
+        it 'records the record count up to that point' do
+          transform_status.record_count.should == 4
+        end
+      end
+    end
+  end
+end