ncs_mdes_warehouse 0.10.1 → 0.11.0

data/lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb

@@ -92,7 +92,12 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
         else
           # node is the start tag of a table variable
           var = node.local_name.to_sym
-          val = node.inner_xml.strip.gsub('
', "\r")
+          val =
+            if node.attribute('xsi:nil') == 'true'
+              nil
+            else
+              node.inner_xml.strip.gsub('
', "\r")
+            end
 
           unless node.self_closing?
             # Skip to closing tag

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.10.1'
+    VERSION = '0.11.0'
   end
 end

data/lib/ncs_navigator/warehouse/xml_emitter.rb

@@ -19,16 +19,18 @@ module NcsNavigator::Warehouse
     #   emitter.
     attr_reader :configuration
 
-    ##
-    # @return [Pathname] the file to which the XML will be emitted.
-    attr_reader :filename
-
     ##
     # @return [Array<Models::MdesModel>] the models whose data will be
     #   emitted. This is determined from the `:tables` option to
     #   {#initialize}.
     attr_reader :models
 
+    ##
+    # @private exposed for testing
+    # @return [Array] the configuration objects related to each separate XML
+    #   file emitted in one run.
+    attr_reader :xml_files
+
     def_delegators :@configuration, :shell, :log
 
     HEADER_TEMPLATE = ERB.new(<<-XML_ERB)
@@ -99,18 +101,13 @@ XML
     #   produced alongside the XML file?
     def initialize(config, filename, options={})
       @configuration = config
-      @include_pii = options[:'include-pii']
-      @filename = case filename
-                  when Pathname
-                    filename
-                  when nil
-                    self.class.default_filename(configuration, @include_pii)
-                  else
-                    Pathname.new(filename.to_s)
-                  end
       @record_count = 0
       @block_size = options[:'block-size'] || 5000
       @zip = options.has_key?(:zip) ? options[:zip] : true
+
+      @xml_files = determine_files_to_create(filename, options)
+
+
       @models =
         if options[:tables]
           options[:tables].collect { |t| t.to_s }.collect { |t|
@@ -126,70 +123,108 @@ XML
     #
     # @return [void]
     def emit_xml
-      shell.say_line("Exporting to #{filename}#{include_pii? ? ' with PII' : ''}")
-      log.info("Beginning XML export to #{filename}")
+      shell.say_line("Exporting to #{xml_files.collect(&:describe).join(', ')}")
+      log.info("Beginning XML export to #{xml_files.collect(&:describe).join(', ')}")
 
       @start = Time.now
-      filename.open('w') do |f|
-        f.write HEADER_TEMPLATE.result(binding)
-
-        models.each do |model|
-          shell.clear_line_then_say('Writing XML for %33s' % model.mdes_table_name)
-
-          write_all_xml_for_model(f, model)
-        end
-
-        f.write FOOTER_TEMPLATE
+      xml_files.each { |xf| xf.write HEADER_TEMPLATE.result(binding) }
+      models.each do |model|
+        shell.clear_line_then_say('Writing XML for %33s' % model.mdes_table_name)
+        write_all_xml_for_model(model)
       end
+      xml_files.each { |xf| xf.write FOOTER_TEMPLATE }
+      xml_files.each { |xf| xf.close }
       @end = Time.now
+
       msg = "%d records written in %d seconds (%.1f/sec).\n" % [@record_count, emit_time, emit_rate]
       shell.clear_line_then_say(msg)
       log.info(msg)
 
-      if zip?
-        shell.say_line("Zipping to #{zip_filename}")
-        log.info("Zipping to #{zip_filename}")
-        Zip::ZipFile.open(zip_filename, Zip::ZipFile::CREATE) do |zf|
-          zf.add(filename.basename, filename)
-        end
-        log.info("XML export complete")
+      xml_files.each { |xf| xf.zip_if_desired }
+      log.info("XML export complete")
+    end
+
+    ##
+    # @return [Pathname] the single file to which the XML will be emitted.
+    #   Throws an exception if writing to multiple files.
+    def filename
+      if xml_files.size == 1
+        xml_files.first.filename
+      else
+        fail "Emitting more than one file. Use `xml_files` to interrogate."
       end
     end
 
     ##
-    # Will PII be included in the exported XML?
-    #
-    # @return [Boolean]
+    # @return [Boolean] Will PII be included in the exported XML?
+    #   Throws an exception if writing to multiple files.
     def include_pii?
-      @include_pii
+      if xml_files.size == 1
+        xml_files.first.include_pii?
+      else
+        fail "Emitting more than one file. Use `xml_files` to interrogate."
+      end
     end
 
     ##
-    # Will a ZIP archive be created along with the XML?
+    # Will ZIP archive(s) be created along with the XML?
     #
     # @return [Boolean]
     def zip?
       @zip
     end
 
-    ##
-    # @return [Pathname] the filename for the ZIP archive of the XML,
-    #   if any. Currently this is always {#filename} + '.zip'.
-    # @see #zip?
-    def zip_filename
-      @zip_filename ||= filename.to_s + '.zip'
+    private
+
+    def determine_files_to_create(filename, options)
+      if options[:'and-pii']
+        # two files, one with and one without PII
+        no_pii_filename = select_filename(filename, false, options[:directory])
+        with_pii_filename = Pathname.new(no_pii_filename.to_s.sub(/^(.*?)(\..*)$/, '\1-PII\2'))
+        [
+          [false, no_pii_filename],
+          [true, with_pii_filename]
+        ].collect do |include_pii, fn|
+          XmlFile.new(fn, include_pii, @zip, shell, log)
+        end
+      else
+        # one file, PII determined by --include-pii
+        include_pii = options[:'include-pii']
+        actual_filename = select_filename(filename, include_pii, options[:directory])
+        [
+          XmlFile.new(actual_filename, include_pii, @zip, shell, log)
+        ]
+      end
     end
 
-    private
+    def select_filename(filename, include_pii, directory_for_default_files)
+      if filename && directory_for_default_files
+        fail "It does not make sense to specify both a filename and the :directory option."
+      end
 
-    def write_all_xml_for_model(f, model)
+      case filename
+      when Pathname
+        filename
+      when nil
+        default_name = self.class.default_filename(configuration, include_pii)
+        if directory_for_default_files
+          Pathname.new(directory_for_default_files) + default_name
+        else
+          default_name
+        end
+      else
+        Pathname.new(filename.to_s)
+      end
+    end
+
+    def write_all_xml_for_model(model)
       shell.say(' %20s' % '[loading]')
       count = model.count
       offset = 0
       while offset < count
         shell.back_up_and_say(20, '%20s' % '[loading]')
         model.all(:limit => @block_size, :offset => offset).each do |instance|
-          instance.write_mdes_xml(f, :indent => 3, :margin => 1, :pii => include_pii?)
+          xml_files.each { |xf| xf.write_instance(instance) }
           @record_count += 1
 
           shell.back_up_and_say(20, '%5d (%5.1f/sec)' % [@record_count, emit_rate])
@@ -217,5 +252,52 @@ XML
     def emit_rate
       @record_count / emit_time
     end
+
+    ##
+    # @private
+    #
+    # Encapsulated the data and operations related to one of the files produced
+    # in a run. Consider "related to one of the files" in versus "related to
+    # loading the records".
+    class XmlFile < Struct.new(:filename, :include_pii, :zip, :shell, :log)
+      alias :include_pii? :include_pii
+      alias :zip? :zip
+
+      def describe
+        "#{filename} #{include_pii? ? 'with' : 'without'} PII"
+      end
+
+      def zip_filename
+        @zip_filename ||= filename.to_s + '.zip'
+      end
+
+      def open
+        @handle ||= filename.open('w')
+      end
+      alias :handle :open
+
+      def write(s)
+        handle.write(s)
+      end
+
+      def write_instance(instance)
+        instance.write_mdes_xml(handle, :indent => 3, :margin => 1, :pii => include_pii?)
+      end
+
+      def close
+        @handle && @handle.close
+      end
+
+      def zip_if_desired
+        if zip?
+          shell.say("Zipping to #{zip_filename}")
+          log.info("Zipping to #{zip_filename}")
+          Zip::ZipFile.open(zip_filename, Zip::ZipFile::CREATE) do |zf|
+            zf.add(filename.basename, filename)
+          end
+          shell.clear_line_then_say("Zipped #{zip_filename}.")
+        end
+      end
+    end
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -42,6 +42,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'actionmailer', '~> 3.0'
 
   s.add_dependency 'treetop'
+  s.add_dependency 'rgl'
 
   s.add_development_dependency 'rspec', '~> 2.6'
   s.add_development_dependency 'rake', '~> 0.9.2'

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -43,7 +43,7 @@ module NcsNavigator::Warehouse
 
     describe '#foreign_key_index' do
       it 'provides an index by default' do
-        config.foreign_key_index.should respond_to(:record_and_verify)
+        config.foreign_key_index.should respond_to(:verify_or_defer)
       end
 
       it 'allows the index to be overridden' do

data/spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb

@@ -17,7 +17,7 @@ module NcsNavigator::Warehouse::Transformers
       include ::DataMapper::Resource
 
       property :id, Integer, :key => true
-      property :name, String
+      property :name, String, :length => (1..100)
       property :age, Integer
       belongs_to :sample,
         'NcsNavigator::Warehouse::Transformers::Sample', :child_key => [ :sample_id ], :required => false
@@ -64,7 +64,7 @@ module NcsNavigator::Warehouse::Transformers
         end
 
         subject.transform(transform_status)
-        transform_status.transform_errors.should be_empty
+        transform_status.transform_errors.should == []
       end
 
       describe 'with an invalid instance' do
@@ -122,6 +122,7 @@ module NcsNavigator::Warehouse::Transformers
           records[2].should_receive(:save).and_return(true)
 
           records[1].psu_id = '20000041'
+          records[1].recruit_type = 'A'
 
           subject.transform(transform_status)
         end
@@ -135,11 +136,11 @@ module NcsNavigator::Warehouse::Transformers
         end
 
         it 'records an error' do
-          transform_status.transform_errors.collect(&:record_id).should == ['2']
+          transform_status.transform_errors.collect(&:record_id).uniq.should == ['2']
         end
 
         describe 'the recorded error' do
-          let(:error) { transform_status.transform_errors.first }
+          let(:error) { transform_status.transform_errors.detect { |err| err.message =~ /PSU/ } }
 
           it 'has the correct model class' do
             error.model_class.should == Sample.to_s
@@ -163,28 +164,135 @@ module NcsNavigator::Warehouse::Transformers
           end
         end
 
+        describe 'and another invalidity' do
+          let(:validation_error) {
+            transform_status.transform_errors.detect { |err| err.message =~ /format/i }
+          }
+
+          it 'simultaneously reports an error about each' do
+            validation_error.attribute_name.should == 'recruit_type'
+          end
+        end
+
         let(:expected_saved_record_ids) { [1, 3] }
         let(:expected_not_saved_record_ids) { [2] }
 
         include_examples 'a foreign key index updater'
       end
 
-      describe 'with an unsatisfied foreign key' do
-        let(:error) { transform_status.transform_errors.first }
+      describe 'with an initially unsatisfied foreign key' do
+        let(:unsatisfied) { Subsample.new(:id => 3, :sample_id => 912, :name => '') }
+        let(:satisfier) { Sample.new(:id => unsatisfied.sample_id, :psu_id => '20000030', :name => 'Nine') }
 
-        before do
-          records << Subsample.new(:id => 3, :sample_id => 912)
+        let(:fk_error) {
+          transform_status.transform_errors.detect { |error| error.message =~ /foreign/i }
+        }
+
+        let(:validation_error) {
+          transform_status.transform_errors.detect { |error| error.message =~ /valid/i }
+        }
 
+        before do
           records.each do |m|
             m.stub!(:valid?).and_return(true)
             m.stub!(:save).and_return(true)
           end
 
-          subject.transform(transform_status)
+          records << unsatisfied
+        end
+
+        describe 'when the record is otherwise valid' do
+          before do
+            unsatisfied.name = 'Foo'
+            unsatisfied.should be_valid # setup
+          end
+
+          describe 'and the foreign key is never satisfied' do
+            it 'reports the unsatisfied foreign key' do
+              unsatisfied.stub!(:save).and_return(true)
+              subject.transform(transform_status)
+
+              [fk_error.attribute_name, fk_error.attribute_value].should == ['sample_id', '912']
+            end
+
+            it 'does not save the record with the unsatisfied key' do
+              unsatisfied.should_not_receive(:save)
+
+              subject.transform(transform_status)
+            end
+          end
+
+          describe 'and the foreign key is eventually satisfied' do
+
+            before do
+              satisfier.stub!(:valid?).and_return(true)
+              satisfier.stub!(:save).and_return(true)
+              records << satisfier
+            end
+
+            it 'does not report any errors' do
+              transform_status.transform_errors.should == []
+            end
+
+            it 'saves the record with the eventually satisfied key' do
+              satisfier.should_receive(:save)
+
+              subject.transform(transform_status)
+            end
+          end
         end
 
-        it 'reports the unsatisfied foreign key' do
-          error.message.should =~ /Unsatisfied foreign key sample_id=912/
+        describe 'when the record has invalid properties' do
+          before do
+            unsatisfied.name = ''
+            unsatisfied.should_not be_valid # setup
+          end
+
+          describe 'and the foreign key is never satisfied' do
+            it 'reports the unsatisfied foreign key' do
+              subject.transform(transform_status)
+
+              [fk_error.attribute_name, fk_error.attribute_value].should == ['sample_id', '912']
+            end
+
+            it 'reports the invalid values' do
+              subject.transform(transform_status)
+
+              validation_error.attribute_name.should == 'name'
+            end
+
+            it 'does not save the record with the unsatisfied key' do
+              unsatisfied.should_not_receive(:save)
+
+              subject.transform(transform_status)
+            end
+          end
+
+          describe 'and the foreign key is eventually satisfied' do
+            before do
+              satisfier.stub!(:valid?).and_return(true)
+              satisfier.stub!(:save).and_return(true)
+              records << satisfier
+            end
+
+            it 'does not report a foreign key problem' do
+              subject.transform(transform_status)
+
+              fk_error.should be_nil
+            end
+
+            it 'does report the invalid value' do
+              subject.transform(transform_status)
+
+              validation_error.attribute_name.should == 'name'
+            end
+
+            it 'does not save the record' do
+              unsatisfied.should_not_receive(:save)
+
+              subject.transform(transform_status)
+            end
+          end
         end
       end