ncs_mdes_warehouse 0.10.1 → 0.11.0

data/spec/ncs_navigator/warehouse/transformers/foreign_key_index_spec.rb

@@ -19,6 +19,32 @@ module NcsNavigator::Warehouse::Transformers
       property :f_id, Integer, :key => true
     end
 
+    class Subcircle1
+      include ::DataMapper::Resource
+
+      property :id, Integer, :key => true
+      belongs_to :subcircle2, 'NcsNavigator::Warehouse::Transformers::Subcircle2',
+        :required => false, :child_key => [ :subcircle2_id ]
+
+      belongs_to :addr, 'NcsNavigator::Warehouse::Transformers::Addr', :child_key => [ :addr_id ]
+    end
+
+    class Subcircle2
+      include ::DataMapper::Resource
+
+      property :id, Integer, :key => true
+      belongs_to :subcircle3, 'NcsNavigator::Warehouse::Transformers::Subcircle3',
+        :required => false, :child_key => [ :subcircle3_id ]
+    end
+
+    class Subcircle3
+      include ::DataMapper::Resource
+
+      property :id, Integer, :key => true
+      belongs_to :subcircle1, 'NcsNavigator::Warehouse::Transformers::Subcircle1',
+        :required => false, :child_key => [ :subcircle1_id ]
+    end
+
     before(:each) do
       DataMapper.finalize
     end
@@ -28,6 +54,13 @@ module NcsNavigator::Warehouse::Transformers
     let(:transform_status) { NcsNavigator::Warehouse::TransformStatus.memory_only('test') }
     let(:errors) { transform_status.transform_errors }
 
+    def verify_and_record_if_appropriate(rec, index=fk_index)
+      if v = index.verify_or_defer(rec)
+        index.record(rec)
+      end
+      v
+    end
+
     describe '#initialize' do
       describe ':existing_key_provider' do
         it 'defaults to a database provider' do
@@ -47,45 +80,47 @@ module NcsNavigator::Warehouse::Transformers
 
     describe 'reporting errors' do
       before do
-        fk_index.record_and_verify(Frob.new(:f_id => 8))
-        fk_index.record_and_verify(Frob.new(:f_id => 16))
+        fk_index.start_transform(transform_status)
+        verify_and_record_if_appropriate(Frob.new(:f_id => 8))
+        verify_and_record_if_appropriate(Frob.new(:f_id => 16))
       end
 
       it 'does not report for a key which is satisfied when it is first recorded' do
-        fk_index.record_and_verify(Addr.new(:frob_id => 8, :a_id => 1))
-        fk_index.report_errors(transform_status)
+        verify_and_record_if_appropriate(Addr.new(:frob_id => 8, :a_id => 1))
+        fk_index.end_transform
 
         errors.should == []
       end
 
       it 'does not report for a key which is not initially satisfied but is later' do
-        fk_index.record_and_verify(Addr.new(:frob_id => 4, :a_id => 1))
-        fk_index.record_and_verify(Frob.new(:f_id => 4))
-        fk_index.report_errors(transform_status)
+        verify_and_record_if_appropriate(Addr.new(:frob_id => 4, :a_id => 1))
+        verify_and_record_if_appropriate(Frob.new(:f_id => 4))
+        fk_index.end_transform
 
         errors.should == []
       end
 
       it 'does not report for a key which is provided by the external key provider' do
-        fk_index.record_and_verify(Addr.new(:a_id => 8, :old_one_id => 120))
-        fk_index.report_errors(transform_status)
+        verify_and_record_if_appropriate(Addr.new(:a_id => 8, :old_one_id => 120))
+        fk_index.end_transform
 
         errors.should == []
       end
 
       it 'does not report for a key which is provided by the external key provider when the associated model class has not previously been referenced' do
         another_index = ForeignKeyIndex.new(:existing_key_provider => key_provider)
+        another_index.start_transform(transform_status)
 
-        another_index.record_and_verify(Addr.new(:a_id => 8, :frob_id => 80))
-        another_index.report_errors(transform_status)
+        verify_and_record_if_appropriate(Addr.new(:a_id => 8, :frob_id => 80), another_index)
+        another_index.end_transform
 
         errors.should == []
       end
 
       describe 'when a key is never satisfied' do
         before do
-          fk_index.record_and_verify(Addr.new(:frob_id => 4, :a_id => 1))
-          fk_index.report_errors(transform_status)
+          verify_and_record_if_appropriate(Addr.new(:frob_id => 4, :a_id => 1))
+          fk_index.end_transform
         end
 
         it 'reports the error' do
@@ -102,22 +137,235 @@ module NcsNavigator::Warehouse::Transformers
 
         it 'includes a useful message' do
           errors.first.message.should ==
-            'Unsatisfied foreign key frob_id=4 referencing NcsNavigator::Warehouse::Transformers::Frob.'
+            'Unsatisfied foreign key referencing NcsNavigator::Warehouse::Transformers::Frob.'
         end
 
-        it 'only reports the error once if #report_errors is called multiple times' do
-          fk_index.report_errors(transform_status)
+        it 'includes the referencing attribute name' do
+          errors.first.attribute_name.should == 'frob_id'
+        end
+
+        it 'includes the unsatisfied value' do
+          errors.first.attribute_value.should == '4'
+        end
+
+        it 'only reports the error for the transform in which it occurs' do
+          fk_index.start_transform(NcsNavigator::Warehouse::TransformStatus.memory_only('test2'))
+          fk_index.end_transform
 
           errors.size.should == 1
         end
       end
 
       it 'reports multiple failed references for a single record' do
-        fk_index.record_and_verify(Addr.new(:a_id => 1, :frob_id => 2, :old_one_id => 3))
-        fk_index.report_errors(transform_status)
+        verify_and_record_if_appropriate(Addr.new(:a_id => 1, :frob_id => 2, :old_one_id => 3))
+        fk_index.end_transform
+
+        errors.collect { |e| [e.attribute_name, e.attribute_value] }.sort.should ==
+          [['frob_id', '2'], ['old_one_id', '3']]
+      end
+    end
+
+    describe 'complex deferred foreign key resolution' do
+      # The full set of records produces a consistent graph like this:
+      #  frob1 <- addr1 <- circle11 -> circle21 -> circle31
+      #                       ^                       |
+      #                       \_______________________|
+      # The various test contexts simulate difficulties by presenting the
+      # records in a different order or by removing some of them.
+      let(:frob1) { Frob.new(:f_id => 1) }
+      let(:addr1) { Addr.new(:a_id => 1, :frob_id => 1)}
+      let(:circle11) { Subcircle1.new(:id => 1, :addr_id => 1, :subcircle2_id => 1)}
+      let(:circle21) { Subcircle2.new(:id => 1, :subcircle3_id => 1)}
+      let(:circle31) { Subcircle3.new(:id => 1, :subcircle1_id => 1)}
+
+      before do
+        fk_index.start_transform(transform_status)
+      end
+
+      def record_idents(records)
+        records.collect { |rec| [rec.class.to_s.split(':').last, rec.key.first].join('-') }
+      end
+
+      describe 'nested out-of-order records which are eventually resolvable' do
+        let!(:record_and_verify_results) {
+          circle21.subcircle3_id = nil
+
+          [circle21, circle11, addr1, frob1].each_with_object({}) do |rec, results|
+            results[rec] = verify_and_record_if_appropriate(rec)
+          end
+        }
+
+        it 'defers records with unresolvable FKs' do
+          record_and_verify_results[addr1].should be_false
+        end
+
+        it 'defers records that refer to records with unresolvable FKs' do
+          record_and_verify_results[circle11].should be_false
+        end
+
+        it 'accepts records which are initially resolved' do
+          [circle21, frob1].collect { |rec| record_and_verify_results[rec] }.should == [true, true]
+        end
+
+        describe 'after all records are visited' do
+          let!(:deferred_records) { fk_index.end_transform }
+
+          it 'reports no errors' do
+            errors.should == []
+          end
+
+          it 'returns the records which may now be saved' do
+            record_idents(deferred_records).sort.should == %w(Addr-1 Subcircle1-1)
+          end
+        end
+      end
+
+      describe 'nested out-of-order records which are not all eventually resolvable' do
+        let!(:record_and_verify_results) {
+          [circle21, circle11, addr1, frob1].each_with_object({}) do |rec, results|
+            results[rec] = verify_and_record_if_appropriate(rec)
+          end
+        }
 
-        errors.collect { |e| e.message.match(/foreign key (\S+)/)[1] }.sort.should ==
-          %w(frob_id=2 old_one_id=3)
+        it 'defers records with unresolvable FKs' do
+          [circle21, addr1].collect { |rec| record_and_verify_results[rec] }.should == [false, false]
+        end
+
+        it 'defers records that refer to records with unresolvable FKs' do
+          record_and_verify_results[circle11].should be_false
+        end
+
+        it 'accepts records which are initially resolved' do
+          record_and_verify_results[frob1].should be_true
+        end
+
+        describe 'after all records are visited' do
+          let!(:deferred_records) { fk_index.end_transform }
+
+          it 'returns the records which may now be saved' do
+            record_idents(deferred_records).should == %w(Addr-1)
+          end
+
+          it 'reports errors for the remaining unresolvable items' do
+            errors.collect(&:model_class).sort.should == [Subcircle1.to_s, Subcircle2.to_s]
+          end
+
+          describe 'the error for a record which has an unresolvable FK' do
+            let(:the_error) { errors.find { |e| e.model_class == Subcircle2.to_s } }
+
+            it 'reports the error' do
+              the_error.message.should == 'Unsatisfied foreign key referencing NcsNavigator::Warehouse::Transformers::Subcircle3.'
+            end
+
+            it 'includes the attribute name' do
+              the_error.attribute_name.should == 'subcircle3_id'
+            end
+
+            it 'includes the attribute value' do
+              the_error.attribute_value.should == '1'
+            end
+          end
+
+          describe 'the error for a record which refers to a record that has an unresolvable FK' do
+            let(:the_error) { errors.find { |e| e.model_class == Subcircle1.to_s } }
+
+            it 'reports the error' do
+              the_error.message.should == 'Associated NcsNavigator::Warehouse::Transformers::Subcircle2 record contains one or more unsatisfed foreign keys or refers to other records that do.'
+            end
+
+            it 'includes the attribute name' do
+              the_error.attribute_name.should == 'subcircle2_id'
+            end
+
+            it 'includes the attribute value' do
+              the_error.attribute_value.should == '1'
+            end
+          end
+        end
+      end
+
+      describe 'circular records which are eventually resolvable' do
+        let!(:record_and_verify_results) {
+          [frob1, circle21, circle11, addr1, circle31].each_with_object({}) do |rec, results|
+            results[rec] = verify_and_record_if_appropriate(rec)
+          end
+        }
+
+        it 'defers all the records in the circle' do
+          [circle11, circle21, circle31].collect { |rec| record_and_verify_results[rec] }.should == [false] * 3
+        end
+
+        it 'accepts records which are initially resolved' do
+          [frob1, addr1].collect { |rec| record_and_verify_results[rec] }.should == [true] * 2
+        end
+
+        describe 'after all records are visited' do
+          let!(:deferred_records) { fk_index.end_transform }
+
+          it 'reports no errors' do
+            errors.should == []
+          end
+
+          it 'returns all the records in the circle' do
+            record_idents(deferred_records).sort.should == %w(Subcircle1-1 Subcircle2-1 Subcircle3-1)
+          end
+        end
+      end
+
+      describe 'circular records which are not eventually resolvable' do
+        let!(:record_and_verify_results) {
+          [circle21, circle11, circle31].each_with_object({}) do |rec, results|
+            results[rec] = verify_and_record_if_appropriate(rec)
+          end
+        }
+
+        it 'defers all the records in the circle' do
+          [circle11, circle21, circle31].collect { |rec| record_and_verify_results[rec] }.should == [false] * 3
+        end
+
+        describe 'after all records are visited' do
+          let!(:deferred_records) { fk_index.end_transform }
+
+          it 'reports an error for each record in the circle, including one for the bad external ref' do
+            errors.collect(&:model_class).sort.should == [Subcircle1.to_s, Subcircle1.to_s, Subcircle2.to_s, Subcircle3.to_s]
+          end
+
+          describe 'the error for the element with the unresolvable FK' do
+            let(:the_error) { errors.find { |e| e.model_class == Subcircle1.to_s && e.attribute_name == 'addr_id' } }
+
+            it 'reports the error' do
+              the_error.message.should == 'Unsatisfied foreign key referencing NcsNavigator::Warehouse::Transformers::Addr.'
+            end
+
+            it 'includes the attribute name' do
+              the_error.attribute_name.should == 'addr_id'
+            end
+
+            it 'includes the attribute value' do
+              the_error.attribute_value.should == '1'
+            end
+          end
+
+          describe 'an error for an element in the circle' do
+            let(:the_error) { errors.find { |e| e.model_class == Subcircle3.to_s } }
+
+            it 'reports the error' do
+              the_error.message.should == 'Associated NcsNavigator::Warehouse::Transformers::Subcircle1 record contains one or more unsatisfed foreign keys or refers to other records that do.'
+            end
+
+            it 'includes the attribute name' do
+              the_error.attribute_name.should == 'subcircle1_id'
+            end
+
+            it 'includes the attribute value' do
+              the_error.attribute_value.should == '1'
+            end
+          end
+
+          it 'returns nothing' do
+            deferred_records.should == []
+          end
+        end
       end
     end
   end

data/spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml

@@ -62,7 +62,7 @@
       <sex>-6</sex>
       <age>49</age>
       <age_range>5</age_range>
-      <person_dob>1962-01-01</person_dob>
+      <person_dob xsi:nil="true"/>
       <deceased>2</deceased>
       <ethnic_group>2</ethnic_group>
       <person_lang>1</person_lang>

data/spec/ncs_navigator/warehouse/transformers/vdr_xml/reader_spec.rb

@@ -50,6 +50,10 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
       it 'strips left over encoded CRs (#1940)' do
         person.person_comment.should == "Likes\r\nline\r\nbreaks"
       end
+
+      it 'reads xsi:nil as nil' do
+        person.person_dob.should be_nil
+      end
     end
   end
 end

data/spec/ncs_navigator/warehouse/xml_emitter_spec.rb

@@ -5,9 +5,10 @@ require 'zip/zip'
 module NcsNavigator::Warehouse
   describe XmlEmitter, :use_mdes do
     let(:filename) { tmpdir + 'export.xml' }
-    let(:options) { {} }
+    let(:options) { { :zip => false } }
+    let(:emitter) { XmlEmitter.new(spec_config, filename, options) }
     let(:xml) {
-      XmlEmitter.new(spec_config, filename, options).emit_xml
+      emitter.emit_xml
       Nokogiri::XML(File.read(filename))
     }
 
@@ -110,7 +111,7 @@ module NcsNavigator::Warehouse
           xml.xpath('//person/person_id').collect { |e| e.text.strip }.sort.should == %w(QX9 XQ4)
         end
 
-        describe 'and PII' do
+        describe 'and including PII in a single file' do
           let(:xml_first_names) {
             xml.xpath('//person/first_name').collect { |e| e.text.strip }.sort
           }
@@ -130,6 +131,39 @@ module NcsNavigator::Warehouse
           end
         end
 
+        describe 'and creating both with- and without-PII variants in parallel' do
+          let(:no_pii_file) { emitter.xml_files.find { |xf| !xf.include_pii? }.filename }
+          let(:pii_file) { emitter.xml_files.find { |xf| xf.include_pii? }.filename }
+
+          let(:no_pii_xml) { Nokogiri::XML(no_pii_file.read) }
+          let(:pii_xml) { Nokogiri::XML(pii_file.read) }
+
+          def first_names_in(xml)
+            xml.xpath('//person/first_name').collect { |e| e.text.strip }.sort
+          end
+
+          before do
+            options[:'and-pii'] = true
+            emitter.emit_xml
+          end
+
+          it 'creates the expected with-PII file' do
+            pii_file.exist?.should be_true
+          end
+
+          it 'creates the expected without-PII file' do
+            no_pii_file.exist?.should be_true
+          end
+
+          it 'includes PII in the PII file' do
+            first_names_in(pii_xml).should == %w(Quentin Xavier)
+          end
+
+          it 'does not include PII in the no-PII file' do
+            first_names_in(no_pii_xml).should == ['', '']
+          end
+        end
+
         describe 'and selected output' do
           let(:people_count) { xml.xpath('//person').size }
           let(:p_count) { xml.xpath('//participant').size }
@@ -177,6 +211,7 @@ module NcsNavigator::Warehouse
 
     describe 'the generated ZIP file', :slow do
       let(:expected_zipfile) { Pathname.new(filename.to_s + '.zip') }
+      let(:options) { {} }
 
       before do
         stub_model(person_model)
@@ -256,5 +291,76 @@ module NcsNavigator::Warehouse
         subject.should be_a Pathname
       end
     end
+
+    describe 'generating PII XML in parallel' do
+      let(:emitter) { XmlEmitter.new(spec_config, provided_filename, options) }
+      let(:options) { { :'and-pii' => true } }
+      let(:provided_filename) { 'emitted.xml' }
+
+      let(:xml_files) { emitter.xml_files }
+      let(:pii_xml_file) { emitter.xml_files.find { |xf| xf.include_pii? } }
+      let(:no_pii_xml_file) { emitter.xml_files.find { |xf| !xf.include_pii? } }
+
+      describe 'filenames' do
+        describe 'when none specified', :slow, :use_mdes do
+          let(:provided_filename) { nil }
+
+          before do
+            NcsNavigator.configuration.psus.first.id = '20000216'
+
+            # Time.parse uses Time.now internally, so this needs to be
+            # defined before starting to register the mock.
+            t = Time.parse('2011-07-28')
+            Time.stub!(:now).and_return(t)
+          end
+
+          it 'uses the normal one for the non-PII variant' do
+            no_pii_xml_file.filename.to_s.should == 'bear_lake-20110728.xml'
+          end
+
+          it 'uses the -PII variant for the one with PII' do
+            pii_xml_file.filename.to_s.should == 'bear_lake-20110728-PII.xml'
+          end
+
+          describe 'but a directory is specified' do
+            before do
+              options[:directory] = '/baz/zap'
+            end
+
+            it 'includes the directory in the with-PII name' do
+              pii_xml_file.filename.to_s.should == '/baz/zap/bear_lake-20110728-PII.xml'
+            end
+
+            it 'includes the directory in the without-PII name' do
+              no_pii_xml_file.filename.to_s.should == '/baz/zap/bear_lake-20110728.xml'
+            end
+          end
+        end
+
+        describe 'when one is specified' do
+          let(:provided_filename) { 'emitted.xml' }
+
+          it 'uses the specified name for the non-PII variant' do
+            no_pii_xml_file.filename.to_s.should == 'emitted.xml'
+          end
+
+          it 'it adds a -PII infix before the extension for the PII variant' do
+            pii_xml_file.filename.to_s.should == 'emitted-PII.xml'
+          end
+        end
+      end
+
+      it 'produces two files' do
+        emitter.xml_files.size.should == 2
+      end
+
+      it 'produces a with-PII file' do
+        pii_xml_file.should_not be_nil
+      end
+
+      it 'produces a without-PII file' do
+        no_pii_xml_file.should_not be_nil
+      end
+    end
   end
 end