ncs_mdes_warehouse 0.12.0 → 0.13.0

data/spec/ncs_navigator/warehouse/contents_spec.rb

@@ -0,0 +1,166 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse
+  describe Contents, :use_mdes do
+    let(:enumerator) { Contents.new(spec_config, options) }
+    let(:options) { { } }
+    let(:yielded) { enumerator.to_a }
+    let(:yielded_keys) { yielded.collect { |i| i.key.first } }
+
+    def person_model
+      spec_config.models_module.const_get(:Person)
+    end
+
+    def participant_model
+      spec_config.models_module.const_get(:Participant)
+    end
+
+    def stub_model(model)
+      model.stub!(:count).and_return(0)
+    end
+
+    before do
+      spec_config.models_module.mdes_order.reject { |m|
+        [person_model, participant_model].include?(m)
+      }.each do |model|
+        stub_model(model)
+      end
+    end
+
+    describe '#each', :slow, :use_database do
+      def default_required_attributes(model)
+        model.properties.select { |prop| prop.required? }.inject({}) { |h, prop|
+          h[prop.name] = '-4'; h
+        }
+      end
+
+      def create_instance(model, attributes)
+        model.new(default_required_attributes(model).merge(attributes))
+      end
+
+      def create_person(id, attributes={})
+        create_instance(person_model, { :person_id => id }.merge(attributes))
+      end
+
+      def yielded_for_model(name)
+        yielded.select { |e| e.class.name =~ /#{name}\z/ }
+      end
+
+      before do
+        records.each { |rec| rec.save or fail "Save of #{rec.inspect} failed." }
+      end
+
+      describe 'with exactly one actual record' do
+        let(:records) {
+          [
+            create_person('XQ4')
+          ]
+        }
+
+        it 'contains records for all models' do
+          enumerator.to_a.size.should == 1
+        end
+
+        it 'contains the right records' do
+          enumerator.collect { |e| e.person_id }.should == %w(XQ4)
+        end
+      end
+
+      describe 'with actual data' do
+        let(:records) {
+          [
+            create_person('XQ4', :first_name => 'Xavier'),
+            create_person('QX9', :first_name => 'Quentin'),
+            create_instance(participant_model, :p_id => 'P_QX4')
+          ]
+        }
+
+        it 'contains records for all models' do
+          enumerator.to_a.size.should == 3
+        end
+
+        it 'contains the right records' do
+          enumerator.collect { |e| e.key.first }.sort.should == %w(P_QX4 QX9 XQ4)
+        end
+
+        describe 'and selected output' do
+          let(:people_count) { yielded_for_model('Person').size }
+          let(:p_count) { yielded_for_model('Participant').size }
+
+          it 'includes all tables by default' do
+            people_count.should == 2
+            p_count.should == 1
+          end
+
+          it 'includes only the selected tables when requested' do
+            options[:tables] = %w(participant)
+
+            people_count.should == 0
+            p_count.should == 1
+          end
+
+          it 'includes all the requested tables when explicitly requested' do
+            options[:tables] = spec_config.models_module.mdes_order.collect(&:mdes_table_name)
+
+            people_count.should == 2
+            p_count.should == 1
+          end
+        end
+
+        describe 'and filters' do
+          it 'yields nothing when the filter removes a record' do
+            options[:filters] = [
+              lambda { |recs| recs.first.key.first == 'QX9' ? [] : recs }
+            ]
+
+            yielded_keys.should == %w(XQ4 P_QX4)
+          end
+
+          it 'yields the replacement record when a record is replaced' do
+            options[:filters] = [
+              lambda { |recs| recs.first.key.first == 'QX9' ? [create_person('AB8')] : recs }
+            ]
+
+            yielded_keys.should == %w(AB8 XQ4 P_QX4)
+          end
+
+          it 'yields all the replacement records when a record is replaced' do
+            options[:filters] = [
+              lambda { |recs| recs.first.key.first == 'QX9' ? [create_person('AB8'), create_person('FR2')] : recs }
+            ]
+
+            yielded_keys.should == %w(AB8 FR2 XQ4 P_QX4)
+          end
+
+          it 'applies the filters in order' do
+            options[:filters] = [
+              lambda { |recs| recs.first.key.first == 'QX9' ? [create_person('AB8', :first_name => 'Adam')] : recs },
+              lambda { |recs| recs.each { |rec| rec.first_name = rec.first_name.reverse if rec.respond_to?(:first_name) } }
+            ]
+
+            yielded.select { |rec| rec.respond_to?(:first_name) }.collect(&:first_name).
+              should == %w(madA reivaX)
+          end
+        end
+      end
+
+      describe 'with lots and lots of actual data' do
+        let(:count) { 3134 }
+        let(:records) { (0...count).collect { |n| create_person(n) } }
+        let(:actual_ids) { enumerator.collect { |e| e.key.first } }
+
+        before do
+          options[:batch_size] = 150
+        end
+
+        it 'contains all the records' do
+          actual_ids.size.should == count
+        end
+
+        it 'contains the right records' do
+          actual_ids.collect(&:to_i).sort[2456].should == 2456
+        end
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/filters/add_id_prefix_filter_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Filters
+  describe AddIdPrefixFilter do
+    let(:filter) {
+      AddIdPrefixFilter.new(spec_config, options)
+    }
+
+    let(:options) {
+      {
+        :table => :participant,
+        :prefix => 'PfX-'
+      }
+    }
+
+    describe '#initialize' do
+      it 'objects if there is no prefix' do
+        options.delete(:prefix)
+
+        expect { filter }.to raise_error('Please specify a :prefix.')
+      end
+
+      it 'does not object if there is a prefix' do
+        expect { filter }.to_not raise_error
+      end
+
+      it 'objects if there is no table or model' do
+        options.delete(:table)
+
+        expect { filter }.to raise_error('Please specify either :table or :model.')
+      end
+
+      it 'accepts a known table' do
+        options[:table] = :link_person_participant
+
+        filter.model.should == spec_config.model(:LinkPersonParticipant)
+      end
+
+      it 'accepts a known model' do
+        options.delete(:table)
+        options[:model] = :LinkContact
+
+        filter.model.should == spec_config.model(:link_contact)
+      end
+    end
+
+    describe '#call' do
+      it 'prefixes the PK on a record of the specified type' do
+        a_p = spec_config.model(:Participant).new(:p_id => '123')
+        filter.call([a_p]).should == [a_p]
+        a_p.p_id.should == 'PfX-123'
+      end
+
+      it 'does not prefix the PK on records of other types' do
+        a_contact = spec_config.model(:Contact).new(:contact_id => '234')
+        filter.call([a_contact]).should == [a_contact]
+        a_contact.contact_id.should == '234'
+      end
+
+      it 'prefixes the FK on references to the specified type' do
+        a_link = spec_config.model(:LinkPersonParticipant).new(
+          :p_id => '345', :person_id => '456')
+        filter.call([a_link]).should == [a_link]
+        a_link.p_id.should == 'PfX-345'
+      end
+
+      it 'does not prefix when the FK is nil' do
+        a_link = spec_config.model(:LinkPersonParticipant).new(
+          :p_id => nil, :person_id => '456')
+        filter.call([a_link]).should == [a_link]
+        a_link.p_id.should be_nil
+      end
+
+      it 'does not prefix an FK which references a different type' do
+        a_link = spec_config.model(:LinkPersonParticipant).new(
+          :p_id => '345', :person_id => '456')
+        filter.call([a_link]).should == [a_link]
+        a_link.person_id.should == '456'
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/{transformers → filters}/apply_global_values_filter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   describe ApplyGlobalValuesFilter, :use_mdes do
     let(:filter) {
       ApplyGlobalValuesFilter.new(spec_config, :values => { :first_name => 'Fred' })

data/spec/ncs_navigator/warehouse/{transformers → filters}/coded_as_missing_filter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   describe CodedAsMissingFilter, :use_mdes do
     def call(records)
       filter.call(records)

data/spec/ncs_navigator/warehouse/{transformers/filters_spec.rb → filters/composite_filter_spec.rb}

@@ -1,25 +1,25 @@
 require 'spec_helper'
 
-module NcsNavigator::Warehouse::Transformers
-  describe Filters do
+module NcsNavigator::Warehouse::Filters
+  describe CompositeFilter do
     let(:filter_one)   { StubFilter.new('one') }
     let(:filter_two)   { StubFilter.new('two') }
     let(:filter_three) { StubFilter.new('three') }
 
     describe '#initialize' do
       it 'sets the #filters attribute' do
-        Filters.new([filter_two, filter_one]).filters.should == [filter_two, filter_one]
+        CompositeFilter.new([filter_two, filter_one]).filters.should == [filter_two, filter_one]
       end
 
       it 'rejects objects that do not have a call method' do
-        lambda { Filters.new([filter_three, Object.new]) }.
+        lambda { CompositeFilter.new([filter_three, Object.new]) }.
           should raise_error(/Filter 1 \(Object\) does not have a call method/)
       end
     end
 
     describe '#call' do
       describe 'with some filters' do
-        let(:filters) { Filters.new([filter_one, filter_two, filter_three]) }
+        let(:filters) { CompositeFilter.new([filter_one, filter_two, filter_three]) }
 
         it 'feeds the result from the each into the next' do
           filter_one.should_receive(:call).with([:foo]).and_return([:foo, :bar])
@@ -50,7 +50,7 @@ module NcsNavigator::Warehouse::Transformers
       end
 
       describe 'with no filters' do
-        let(:filters) { Filters.new([]) }
+        let(:filters) { CompositeFilter.new([]) }
 
         it 'returns the input' do
           filters.call([:foo]).should == [:foo]
@@ -67,10 +67,10 @@ module NcsNavigator::Warehouse::Transformers
     end
 
     describe 'enumerableness' do
-      let(:filters) { Filters.new([filter_one, filter_two, filter_three]) }
+      let(:filters) { CompositeFilter.new([filter_one, filter_two, filter_three]) }
 
       it 'is Enumerable' do
-        Filters.ancestors.should include(::Enumerable)
+        CompositeFilter.ancestors.should include(::Enumerable)
       end
 
       it 'delegates #each to the filter list' do

data/spec/ncs_navigator/warehouse/{transformers → filters}/no_blank_foreign_keys_filter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   describe NoBlankForeignKeysFilter, :use_mdes do
     describe '.call' do
       def call(records)

data/spec/ncs_navigator/warehouse/{transformers → filters}/no_ssu_outreach_all_ssus_filter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   describe NoSsuOutreachAllSsusFilter, :use_mdes do
     describe '#initialize' do
       it 'defaults the SSU list to the configuration list' do

data/spec/ncs_navigator/warehouse/{transformers → filters}/no_ssu_outreach_placeholder_filter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   describe NoSsuOutreachPlaceholderFilter, :use_mdes do
     let(:filter) { NoSsuOutreachPlaceholderFilter.new(spec_config) }
 

data/spec/ncs_navigator/warehouse/filters/remove_id_prefix_filter_spec.rb

@@ -0,0 +1,95 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Filters
+  describe RemoveIdPrefixFilter do
+    let(:filter) {
+      RemoveIdPrefixFilter.new(spec_config, options)
+    }
+
+    let(:options) {
+      {
+        :table => :participant,
+        :prefix => 'ZAP-'
+      }
+    }
+
+    describe '#initialize' do
+      it 'objects if there is no prefix' do
+        options.delete(:prefix)
+
+        expect { filter }.to raise_error('Please specify a :prefix.')
+      end
+
+      it 'does not object if there is a prefix' do
+        expect { filter }.to_not raise_error
+      end
+
+      it 'objects if there is no table or model' do
+        options.delete(:table)
+
+        expect { filter }.to raise_error('Please specify either :table or :model.')
+      end
+
+      it 'accepts a known table' do
+        options[:table] = :link_person_participant
+
+        filter.model.should == spec_config.model(:LinkPersonParticipant)
+      end
+
+      it 'accepts a known model' do
+        options.delete(:table)
+        options[:model] = :LinkContact
+
+        filter.model.should == spec_config.model(:link_contact)
+      end
+    end
+
+    describe '#call' do
+      it 'removes the prefix from a record of the specified type' do
+        a_p = spec_config.model(:Participant).new(:p_id => 'ZAP-123')
+        filter.call([a_p]).should == [a_p]
+        a_p.p_id.should == '123'
+      end
+
+      it 'does not remove the prefix if it is not present' do
+        a_p = spec_config.model(:Participant).new(:p_id => '321')
+        filter.call([a_p]).should == [a_p]
+        a_p.p_id.should == '321'
+      end
+
+      it 'does not touch a prefix on the PK on records of other types' do
+        a_contact = spec_config.model(:Contact).new(:contact_id => 'ZAP-234')
+        filter.call([a_contact]).should == [a_contact]
+        a_contact.contact_id.should == 'ZAP-234'
+      end
+
+      it 'removes the prefix from the FK on references to the specified type' do
+        a_link = spec_config.model(:LinkPersonParticipant).new(
+          :p_id => 'ZAP-345', :person_id => 'ZAP-456')
+        filter.call([a_link]).should == [a_link]
+        a_link.p_id.should == '345'
+      end
+
+      it 'does not remove the prefix when it is not set' do
+        a_link = spec_config.model(:LinkPersonParticipant).new(
+          :p_id => '543', :person_id => '654')
+        filter.call([a_link]).should == [a_link]
+        a_link.p_id.should == '543'
+      end
+
+      it 'does not remove the prefix from the FK which references a different type' do
+        a_link = spec_config.model(:LinkPersonParticipant).new(
+          :p_id => 'ZAP-345', :person_id => 'ZAP-456')
+        filter.call([a_link]).should == [a_link]
+        a_link.person_id.should == 'ZAP-456'
+      end
+
+      it 'does not fail when the FK is nil' do
+        a_link = spec_config.model(:LinkPersonParticipant).new(
+          :p_id => nil, :person_id => '654')
+        filter.call([a_link]).should == [a_link]
+        a_link.p_id.should be_nil
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/xml_emitter_spec.rb

@@ -187,6 +187,81 @@ module NcsNavigator::Warehouse
             p_count.should == 1
           end
         end
+
+        describe 'and filters' do
+          let(:person_names) { xml.xpath('//person/first_name').collect(&:inner_text) }
+
+          before do
+            options[:'include-pii'] = true
+
+            spec_config.add_filter_set :reverser, lambda { |recs|
+              if recs.first.respond_to?(:first_name)
+                recs.first.first_name = recs.first.first_name.reverse
+              end
+              recs
+            }
+            spec_config.add_filter_set :upcaser, lambda { |recs|
+              if recs.first.respond_to?(:first_name)
+                recs.first.first_name = recs.first.first_name.upcase
+              end
+              recs
+            }
+          end
+
+          after do
+            # spec_config is shared across all specs, so clean it up
+            spec_config.filter_sets.delete(:reverser)
+            spec_config.filter_sets.delete(:upcaser)
+          end
+
+          describe 'when there is a default XML filter in the configuration' do
+            before do
+              spec_config.default_xml_filter_set = :reverser
+            end
+
+            after do
+              spec_config.default_xml_filter_set = nil
+            end
+
+            it 'is used when there are no filters in the options' do
+              options.delete(:filters)
+
+              person_names.should == %w(nitneuQ reivaX)
+            end
+
+            it 'is not used when a different filter is in the options' do
+              options[:filters] = ['upcaser']
+
+              person_names.should == %w(QUENTIN XAVIER)
+            end
+
+            it 'is not used when explicitly disabled' do
+              options[:filters] = nil
+
+              person_names.should == %w(Quentin Xavier)
+            end
+          end
+
+          describe 'when there is no default XML filter in the configuration' do
+            it 'applies no filters when none are specified' do
+              options.delete(:filters)
+
+              person_names.should == %w(Quentin Xavier)
+            end
+
+            it 'applies no filters when an absence of filters is specified' do
+              options[:filters] = nil
+
+              person_names.should == %w(Quentin Xavier)
+            end
+
+            it 'applies specified filters, if any' do
+              options[:filters] = ['upcaser', 'reverser']
+
+              person_names.should == %w(NITNEUQ REIVAX)
+            end
+          end
+        end
       end
 
       describe 'with lots and lots of actual data', :slow, :use_database do
@@ -207,6 +282,25 @@ module NcsNavigator::Warehouse
           actual_ids.collect(&:to_i).sort[2456].should == 2456
         end
       end
+
+      describe 'with directly provided instances' do
+        before do
+          options[:content] = [
+            create_person('XQ9', :first_name => 'Xavier'),
+            create_person('QX4', :first_name => 'Quentin'),
+            create_instance(participant_model, :p_id => 'P_QX9')
+          ]
+        end
+
+        it 'contains the specified records' do
+          xml.xpath('//person').size.should == 2
+          xml.xpath('//participant').size.should == 1
+        end
+
+        it 'contains the records in the provided order' do
+          xml.xpath('//person/person_id').collect { |e| e.text.strip }.should == %w(XQ9 QX4)
+        end
+      end
     end
 
     describe 'the generated ZIP file', :slow do