ncs_mdes_warehouse 0.1.1 → 0.2.0

data/lib/ncs_navigator/warehouse/transform_status.rb

@@ -41,7 +41,10 @@ module NcsNavigator::Warehouse
 
     def unsuccessful_record(record, message)
       self.transform_errors <<
-        TransformError.new(:model_class => record.class.name, :message => message)
+        TransformError.new(
+          :model_class => record.class.name,
+          :record_id => (record.key.first if record && record.key),
+          :message => message)
     end
   end
 
@@ -54,6 +57,7 @@ module NcsNavigator::Warehouse
     property :id,          Serial
     property :message,     Text,   :required => true
     property :model_class, String, :length => 255
+    property :record_id,   String, :length => 255
 
     belongs_to :transform_status, TransformStatus, :required => true
   end

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -37,16 +37,17 @@ module NcsNavigator::Warehouse::Transformers
     # @return [void]
     def transform(status)
       enum.each do |record|
+        apply_global_values_if_necessary(record)
         if record.valid?
           log.debug("Saving valid record #{record_ident record}.")
           begin
             unless record.save
-              msg = "Could not save. #{record_ident(record)}."
+              msg = "Could not save."
               log.error msg
               status.unsuccessful_record(record, msg)
             end
           rescue => e
-            msg = "Error on save. #{e.class}: #{e}. #{record_ident(record)}."
+            msg = "Error on save. #{e.class}: #{e}."
             log.error msg
             status.unsuccessful_record(record, msg)
           end
@@ -57,7 +58,7 @@ module NcsNavigator::Warehouse::Transformers
               "#{e} (#{prop}=#{v.inspect})."
             }
           }.flatten
-          msg = "Invalid record. #{messages.join(' ')} #{record_ident(record)}."
+          msg = "Invalid record. #{messages.join(' ')}"
           log.error msg
           status.unsuccessful_record(record, msg)
         end
@@ -72,5 +73,19 @@ module NcsNavigator::Warehouse::Transformers
       '%s %s=%s' % [
         rec.class.name.demodulize, rec.class.key.first.name, rec.key.try(:first).inspect]
     end
+
+    def apply_global_values_if_necessary(record)
+      {
+        :psu_id => @configuration.navigator.psus.first.id,
+        :recruit_type => @configuration.navigator.recruitment_type_id
+      }.each do |attr, value|
+        setter = :"#{attr}="
+        if record.respond_to?(setter) && record.respond_to?(attr)
+          unless record.send(attr)
+            record.send(setter, value)
+          end
+        end
+      end
+    end
   end
 end

data/lib/ncs_navigator/warehouse/updating_shell.rb

@@ -22,6 +22,7 @@ module NcsNavigator::Warehouse
       say clear_line_chars
       say s
     end
+    alias :clear_line_and_say :clear_line_then_say
 
     def say_line(s)
       say s, "\n"

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.1.1'
+    VERSION = '0.2.0'
   end
 end

data/lib/ncs_navigator/warehouse/xml_emitter.rb

@@ -7,12 +7,28 @@ require 'fileutils'
 require 'forwardable'
 
 module NcsNavigator::Warehouse
+  ##
+  # Generates VDR XML from a warehouse instance. This is the object
+  # which implements the `emit-xml` tool in the `mdes-wh` command line
+  # client.
   class XmlEmitter
     extend Forwardable
 
+    ##
+    # @return [Configuration] the warehouse configuration used by this
+    #   emitter.
     attr_reader :configuration
+
+    ##
+    # @return [Pathname] the file to which the XML will be emitted.
     attr_reader :filename
 
+    ##
+    # @return [Array<Models::MdesModel>] the models whose data will be
+    #   emitted. This is determined from the `:tables` option to
+    #   {#initialize}.
+    attr_reader :models
+
     def_delegators :@configuration, :shell, :log
 
     HEADER_TEMPLATE = ERB.new(<<-XML_ERB)
@@ -39,6 +55,10 @@ XML_ERB
 </ncs:recruitment_substudy_transmission_envelope>
 XML
 
+    ##
+    # @param configuration [Configuration]
+    # @return [Pathname] the default filename to use for a VDR XML
+    #   submission. The format is `{county}-{YYYYMMDD}.xml`.
     def self.default_filename(configuration)
       psu_type = configuration.mdes.types.detect { |type| type.name =~ /^psu_cl/ }
       unless psu_type
@@ -48,7 +68,7 @@ XML
       psu_id = configuration.navigator.psus.first.id
       psu_entry =  psu_type.code_list.detect { |cle| cle.value == psu_id }
       unless psu_entry
-        fail "Cannot find PSU #{psu_id} in #{psu_type.name}. Please specify a filename manually"
+        fail "Cannot find PSU #{psu_id} in #{psu_type.name}. Please specify a filename manually."
       end
 
       Pathname.new '%s-%s.xml' % [
@@ -57,7 +77,26 @@ XML
       ]
     end
 
-    def initialize(config, filename)
+    ##
+    # Create a new {XmlEmitter}.
+    #
+    # @param [Configuration] config the configuration for the
+    #   warehouse from which to emit records.
+    # @param [Pathname,#to_s,nil] filename the filename to which the output
+    #   will be written. If `nil`, the {.default_filename} is used.
+    #
+    # @option options [Fixnum] :block-size (5000) the maximum number
+    #   of records to load into memory before writing them to the XML
+    #   file. Reduce this to reduce the memory load of the emitter.
+    #   Increasing it will probably not improve performance, even if
+    #   you have sufficient memory to load more records.
+    # @option options [Boolean] :include-pii (false) should PII
+    #   variable values be included in the XML?
+    # @option options [Array<#to_s>] :tables (all for current MDES
+    #   version) the tables to include in the emitted XML.
+    # @option options [Boolean] :zip (true) should a ZIP file be
+    #   produced alongside the XML file?
+    def initialize(config, filename, options={})
       @configuration = config
       @filename = case filename
                   when Pathname
@@ -68,8 +107,23 @@ XML
                     Pathname.new(filename.to_s)
                   end
       @record_count = 0
+      @block_size = options[:'block-size'] || 5000
+      @zip = options.has_key?(:zip) ? options[:zip] : true
+      @include_pii = options[:'include-pii']
+      @models =
+        if options[:tables]
+          options[:tables].collect { |t| t.to_s }.collect { |t|
+            config.models_module.mdes_order.find { |model| model.mdes_table_name == t }
+          }
+        else
+          config.models_module.mdes_order
+        end
     end
 
+    ##
+    # Emit XML from the configured warehouse to {#filename}.
+    #
+    # @return [void]
     def emit_xml
       shell.say_line("Exporting to #{filename}")
       log.info("Beginning XML export to #{filename}")
@@ -78,7 +132,7 @@ XML
       filename.open('w') do |f|
         f.write HEADER_TEMPLATE.result(binding)
 
-        configuration.models_module.mdes_order.each do |model|
+        models.each do |model|
           shell.clear_line_then_say('Writing XML for %33s' % model.mdes_table_name)
 
           write_all_xml_for_model(f, model)
@@ -91,14 +145,36 @@ XML
       shell.clear_line_then_say(msg)
       log.info(msg)
 
-      shell.say_line("Zipping to #{zip_filename}")
-      log.info("Zipping to #{zip_filename}")
-      Zip::ZipFile.open(zip_filename, Zip::ZipFile::CREATE) do |zf|
-        zf.add(filename.basename, filename)
+      if zip?
+        shell.say_line("Zipping to #{zip_filename}")
+        log.info("Zipping to #{zip_filename}")
+        Zip::ZipFile.open(zip_filename, Zip::ZipFile::CREATE) do |zf|
+          zf.add(filename.basename, filename)
+        end
+        log.info("XML export complete")
       end
-      log.info("XML export complete")
     end
 
+    ##
+    # Will PII be included in the exported XML?
+    #
+    # @return [Boolean]
+    def include_pii?
+      @include_pii
+    end
+
+    ##
+    # Will a ZIP archive be created along with the XML?
+    #
+    # @return [Boolean]
+    def zip?
+      @zip
+    end
+
+    ##
+    # @return [Pathname] the filename for the ZIP archive of the XML,
+    #   if any. Currently this is always {#filename} + '.zip'.
+    # @see #zip?
     def zip_filename
       @zip_filename ||= filename.to_s + '.zip'
     end
@@ -107,11 +183,17 @@ XML
 
     def write_all_xml_for_model(f, model)
       shell.say(' %20s' % '[loading]')
-      model.all.each do |instance|
-        instance.write_mdes_xml(f, :indent => 3, :margin => 1)
-        @record_count += 1
-
-        shell.back_up_and_say(20, '%5d (%5.1f/sec)' % [@record_count, emit_rate])
+      count = model.count
+      offset = 0
+      while offset < count
+        shell.back_up_and_say(20, '%20s' % '[loading]')
+        model.all(:limit => @block_size, :offset => offset).each do |instance|
+          instance.write_mdes_xml(f, :indent => 3, :margin => 1, :pii => include_pii?)
+          @record_count += 1
+
+          shell.back_up_and_say(20, '%5d (%5.1f/sec)' % [@record_count, emit_rate])
+        end
+        offset += @block_size
       end
     end
 

data/ncs_mdes_warehouse.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib", "generated_models"]
 
-  s.add_dependency 'ncs_mdes', '~> 0.4', '>= 0.4.2'
+  s.add_dependency 'ncs_mdes', '~> 0.5'
   s.add_dependency 'ncs_navigator_configuration', '~> 0.2'
 
   s.add_dependency 'activesupport', '~> 3.0'

data/spec/ncs_navigator/warehouse/data_mapper_spec.rb

@@ -0,0 +1,39 @@
+require File.expand_path('../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse::DataMapper
+  shared_examples 'a string-based NCS type' do
+    it 'converts a BigDecimal to a simple number string (no powers of ten)' do
+      subject.send(:typecast_to_primitive, BigDecimal.new('0.453E1')).should == '4.53'
+    end
+
+    it 'uses the default method for other types' do
+      subject.send(:typecast_to_primitive, 45).should == '45'
+    end
+  end
+
+  describe NcsString, :modifies_warehouse_state do
+    let(:model) {
+      class NcsStringHost
+        include DataMapper::Resource
+        property :haiku, NcsString
+      end
+    }
+
+    subject { model.properties[:haiku] }
+
+    it_behaves_like 'a string-based NCS type'
+  end
+
+  describe NcsText, :modifies_warehouse_state do
+    let(:model) {
+      class NcsTextHost
+        include DataMapper::Resource
+        property :novel, NcsText
+      end
+    }
+
+    subject { model.properties[:novel] }
+
+    it_behaves_like 'a string-based NCS type'
+  end
+end

data/spec/ncs_navigator/warehouse/transform_load_spec.rb

@@ -13,7 +13,9 @@ module NcsNavigator::Warehouse
       end
     end
 
-    before(:all) do
+    before do
+      use_test_bcdatabase
+
       # Not config due to RSpec #500
       DatabaseInitializer.new(base_config).tap do |dbi|
         dbi.set_up_repository(:both)

data/spec/ncs_navigator/warehouse/transformers/database_spec.rb

@@ -48,7 +48,7 @@ module NcsNavigator::Warehouse::Transformers
     end
 
     describe '#connection_parameters' do
-      describe 'from bcdatabase', :modifies_warehouse_state do
+      describe 'from bcdatabase', :modifies_warehouse_state, :use_test_bcdatabase do
         let(:cls) {
           sample_class do
             bcdatabase :name => 'ncs_staff_portal'
@@ -79,7 +79,7 @@ module NcsNavigator::Warehouse::Transformers
       end
     end
 
-    describe '#repository' do
+    describe '#repository', :modifies_warehouse_state, :use_test_bcdatabase do
       include_context 'people_pro'
 
       let(:cls) do
@@ -97,7 +97,7 @@ module NcsNavigator::Warehouse::Transformers
       end
     end
 
-    describe '#each' do
+    describe '#each', :modifies_warehouse_state, :use_test_bcdatabase do
       include_context 'people_pro'
 
       describe 'of .produce_records' do

data/spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb

@@ -5,6 +5,8 @@ module NcsNavigator::Warehouse::Transformers
     class Sample
       include ::DataMapper::Resource
 
+      property :psu_id, String
+      property :recruit_type, String
       property :id, Integer, :key => true
       property :name, String, :required => true
     end
@@ -14,7 +16,11 @@ module NcsNavigator::Warehouse::Transformers
     end
 
     describe '#transform' do
-      let(:config) { NcsNavigator::Warehouse::Configuration.new }
+      let(:config) {
+        NcsNavigator::Warehouse::Configuration.new.tap do |c|
+          c.log_file = tmpdir + 'enum_transformer_test.log'
+        end
+      }
       let(:records) { [
           Sample.new(:id => 1, :name => 'One'),
           Sample.new(:id => 2, :name => 'Two'),
@@ -33,7 +39,29 @@ module NcsNavigator::Warehouse::Transformers
         transform_status.transform_errors.should be_empty
       end
 
-      it 'automatically sets the PSU ID if the object accepts it'
+      it 'automatically sets the PSU ID if necessary' do
+        records[1].psu_id = '20000042'
+
+        records.each do |m|
+          m.should_receive(:valid?).and_return(true)
+          m.should_receive(:save).and_return(true)
+        end
+
+        subject.transform(transform_status)
+        records.collect(&:psu_id).should == %w(20000030 20000042 20000030)
+      end
+
+      it 'automatically sets recruit_type if necessary' do
+        records[2].recruit_type = '1'
+
+        records.each do |m|
+          m.should_receive(:valid?).and_return(true)
+          m.should_receive(:save).and_return(true)
+        end
+
+        subject.transform(transform_status)
+        records.collect(&:recruit_type).should == %w(3 3 1)
+      end
 
       describe 'with an invalid instance' do
         before do
@@ -47,7 +75,8 @@ module NcsNavigator::Warehouse::Transformers
         it 'records the invalid instance' do
           err = transform_status.transform_errors.first
           err.model_class.should == Sample.to_s
-          err.message.should == 'Invalid record. Name must not be blank (name=nil). Sample id=3.'
+          err.record_id.should == '3'
+          err.message.should == 'Invalid record. Name must not be blank (name=nil).'
         end
 
         it 'saves the other instances' do
@@ -67,7 +96,8 @@ module NcsNavigator::Warehouse::Transformers
         it 'records the unsaveable instance' do
           err = transform_status.transform_errors.first
           err.model_class.should == Sample.to_s
-          err.message.should == 'Could not save. Sample id=2.'
+          err.record_id.should == '2'
+          err.message.should == 'Could not save.'
         end
 
         it 'saves the saveable instances' do
@@ -87,8 +117,9 @@ module NcsNavigator::Warehouse::Transformers
         it 'records the failing instance' do
           err = transform_status.transform_errors.first
           err.model_class.should == Sample.to_s
+          err.record_id.should == '1'
           err.message.should ==
-            'Error on save. RuntimeError: No database around these parts. Sample id=1.'
+            'Error on save. RuntimeError: No database around these parts.'
         end
 
         it 'saves the saveable instances' do

data/spec/ncs_navigator/warehouse/xml_emitter_spec.rb

@@ -5,8 +5,9 @@ require 'zip/zip'
 module NcsNavigator::Warehouse
   describe XmlEmitter, :use_mdes do
     let(:filename) { tmpdir + 'export.xml' }
+    let(:options) { {} }
     let(:xml) {
-      XmlEmitter.new(spec_config, filename).emit_xml
+      XmlEmitter.new(spec_config, filename, options).emit_xml
       Nokogiri::XML(File.read(filename))
     }
 
@@ -14,33 +15,91 @@ module NcsNavigator::Warehouse
       spec_config.models_module.const_get(:Person)
     end
 
+    def participant_model
+      spec_config.models_module.const_get(:Participant)
+    end
+
+    def stub_model(model)
+      model.stub!(:count).and_return(0)
+    end
+
     before do
-      spec_config.models_module.mdes_order.each do |model|
-        model.stub!(:all).and_return([])
+      spec_config.models_module.mdes_order.reject { |m|
+        [person_model, participant_model].include?(m)
+      }.each do |model|
+        stub_model(model)
       end
     end
 
     # Most of the details of the XML are tested on the MdesModel mixin
     describe 'the generated XML', :slow do
-      it 'includes the SC from the configuration' do
-        xml.xpath('//sc_id').text.should == '20000029'
+      describe 'global attributes' do
+        before do
+          stub_model(person_model)
+          stub_model(participant_model)
+        end
+
+        it 'includes the SC from the configuration' do
+          xml.xpath('//sc_id').text.should == '20000029'
+        end
+
+        it 'includes the PSU from the configuration' do
+          xml.xpath('//psu_id').text.should == '20000030'
+        end
+
+        it 'includes the appropriate specification version' do
+          xml.xpath('//specification_version').text.
+            should == spec_config.mdes.specification_version
+        end
+      end
+
+      def default_required_attributes(model)
+        model.properties.select { |prop| prop.required? }.inject({}) { |h, prop|
+          h[prop.name] = '-4'; h
+        }
+      end
+
+      def create_instance(model, attributes)
+        model.new(default_required_attributes(model).merge(attributes))
       end
 
-      it 'includes the PSU from the configuration' do
-        xml.xpath('//psu_id').text.should == '20000030'
+      def create_person(id, attributes={})
+        create_instance(person_model, { :person_id => id }.merge(attributes))
       end
 
-      it 'includes the appropriate specification version' do
-        xml.xpath('//specification_version').text.
-          should == spec_config.mdes.specification_version
+      describe 'with exactly one actual record', :slow, :use_database do
+        let(:records) {
+          [
+            create_person('XQ4')
+          ]
+        }
+
+        before do
+          pending 'Not working in CI at the moment' if ENV['CI_RUBY']
+          records.each { |rec| rec.save or fail "Save of #{rec.inspect} failed." }
+        end
+
+        it 'contains records for all models' do
+          xml.xpath('//person').size.should == 1
+        end
+
+        it 'contains the right records' do
+          xml.xpath('//person/person_id').collect { |e| e.text.strip }.sort.should == %w(XQ4)
+        end
       end
 
-      describe 'with actual data' do
+      describe 'with actual data', :slow, :use_database do
+        let(:records) {
+          [
+            create_person('XQ4', :first_name => 'Xavier'),
+            create_person('QX9', :first_name => 'Quentin'),
+            create_instance(participant_model, :p_id => 'P_QX4')
+          ]
+        }
+
         before do
-          person_model.should_receive(:all).and_return([
-              person_model.new(:person_id => 'XQ4'),
-              person_model.new(:person_id => 'QX9')
-            ])
+          pending 'Not working in CI at the moment' if ENV['CI_RUBY']
+          records.each { |rec| rec.save or fail "Save of #{rec.inspect} failed." }
         end
 
         it 'contains records for all models' do
@@ -48,7 +107,70 @@ module NcsNavigator::Warehouse
         end
 
         it 'contains the right records' do
-          xml.xpath('//person/person_id').collect { |e| e.text.strip }.should == %w(XQ4 QX9)
+          xml.xpath('//person/person_id').collect { |e| e.text.strip }.sort.should == %w(QX9 XQ4)
+        end
+
+        describe 'and PII' do
+          let(:xml_first_names) {
+            xml.xpath('//person/first_name').collect { |e| e.text.strip }.sort
+          }
+
+          it 'excludes PII by default' do
+            xml_first_names.should == ['', '']
+          end
+
+          it 'excludes PII when explicitly excluded' do
+            options[:'include-pii'] = false
+            xml_first_names.should == ['', '']
+          end
+
+          it 'includes PII when requested' do
+            options[:'include-pii'] = true
+            xml_first_names.should == %w(Quentin Xavier)
+          end
+        end
+
+        describe 'and selected output' do
+          let(:people_count) { xml.xpath('//person').size }
+          let(:p_count) { xml.xpath('//participant').size }
+
+          it 'includes all tables by default' do
+            people_count.should == 2
+            p_count.should == 1
+          end
+
+          it 'includes only the selected tables when requested' do
+            options[:tables] = %w(participant)
+
+            people_count.should == 0
+            p_count.should == 1
+          end
+
+          it 'includes all the requested tables when explicitly requested' do
+            options[:tables] = spec_config.models_module.mdes_order.collect(&:mdes_table_name)
+
+            people_count.should == 2
+            p_count.should == 1
+          end
+        end
+      end
+
+      describe 'with lots and lots of actual data', :slow, :use_database do
+        let(:count) { 3134 }
+        let(:records) { (0...count).collect { |n| create_person(n) } }
+        let(:actual_ids) { xml.xpath('//person/person_id').collect { |e| e.text.strip } }
+
+        before do
+          pending 'Not working in CI at the moment' if ENV['CI_RUBY']
+          records.each { |rec| rec.save or fail "Save of #{rec.inspect} failed." }
+        end
+
+        it 'contains all the records' do
+          actual_ids.size.should == count
+        end
+
+        it 'contains the right records' do
+          actual_ids.collect(&:to_i).sort[2456].should == 2456
         end
       end
     end
@@ -57,27 +179,51 @@ module NcsNavigator::Warehouse
       let(:expected_zipfile) { Pathname.new(filename.to_s + '.zip') }
 
       before do
-        xml # for side effects
+        stub_model(person_model)
+        stub_model(participant_model)
       end
 
-      it 'exists' do
-        expected_zipfile.should be_readable
-      end
+      context do
+        def actual
+          xml
+          expected_zipfile
+        end
+
+        it 'exists by default' do
+          actual.should be_readable
+        end
 
-      it 'contains just the XML file' do
-        contents = []
-        Zip::ZipFile.foreach(expected_zipfile) do |entry|
-          contents << entry.name
+        it 'exists if explicitly requested' do
+          options[:zip] = true
+          actual.should be_readable
+        end
+
+        it 'does not exist when excluded' do
+          options[:zip] = false
+          actual.exist?.should be_false
         end
-        contents.should == [ filename.basename.to_s ]
       end
 
-      it 'can be opened by something other than rubyzip' do
-        `which unzip`
-        pending "unzip is not available" unless $? == 0
+      context do
+        before do
+          xml # for side effects
+        end
 
-        `unzip -l '#{expected_zipfile}' 2>&1`.should =~ /#{filename.basename}\s/
-        $?.should == 0
+        it 'contains just the XML file' do
+          contents = []
+          Zip::ZipFile.foreach(expected_zipfile) do |entry|
+            contents << entry.name
+          end
+          contents.should == [ filename.basename.to_s ]
+        end
+
+        it 'can be opened by something other than rubyzip' do
+          `which unzip`
+          pending "unzip is not available" unless $? == 0
+
+          `unzip -l '#{expected_zipfile}' 2>&1`.should =~ /#{filename.basename}\s/
+          $?.should == 0
+        end
       end
     end