ncs_mdes_warehouse 0.5.0 → 0.6.0

data/lib/ncs_navigator/warehouse/transformers/vdr_xml.rb

@@ -8,8 +8,8 @@ module NcsNavigator::Warehouse::Transformers
       ##
       # @return [#transform] a transformer that loads the MDES data in
       #   the specified VDR XML file.
-      def from_file(config, filename) # <- TODO better solution
-        EnumTransformer.new(config, Reader.new(config, filename))
+      def from_file(config, filename, options={}) # <- TODO better solution
+        EnumTransformer.new(config, Reader.new(config, filename), options)
       end
 
       ##
@@ -17,10 +17,12 @@ module NcsNavigator::Warehouse::Transformers
       #   warehouse.
       # @param list [String,Array<String>] the files to consider. This
       #   may be either a glob or an explicit list of files.
+      # @param options [Hash<Symbol, Object>] options for the
+      #   resulting transformer.
       #
       # @return [#transform] a transformer for the most recently
       #   modified VDR XML file from the given list of files.
-      def from_most_recent_file(config, list)
+      def from_most_recent_file(config, list, options={})
         files =
           if String === list
             Dir[list].tap { |a| fail "Glob #{list} does not match any files." if a.empty? }
@@ -31,7 +33,8 @@ module NcsNavigator::Warehouse::Transformers
         from_file(
           config,
           files.collect { |fn| [fn, File.stat(fn).mtime] }.
-            sort_by { |fn, mtime| mtime }.reverse.first.first)
+            sort_by { |fn, mtime| mtime }.reverse.first.first,
+          options)
       end
     end
   end

data/lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb

@@ -84,10 +84,7 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
       elsif @current_model_class
         if !is_open && node.local_name == @current_model_class.mdes_table_name
           # on the way out of this record
-          rec = build_current_instance
-          unless MISSING_CODES.include?(rec.key.first)
-            yield rec
-          end
+          yield build_current_instance
           @current_model_class = nil
           @record_count += 1
           shell.clear_line_then_say('%6d records (%3.1f per second); up to %s' %
@@ -121,15 +118,6 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
 
     def should_filter_out(model_class, variable_name, value)
       return true if variable_name == :transaction_type
-
-      if is_foreign_key_in_current_model?(variable_name)
-        return MISSING_CODES.include?(value) || value.strip.empty?
-      end
-
-      property = @current_model_class.properties.find { |p| p.name == variable_name }
-      unless property.required? || property.options[:set]
-        return MISSING_CODES.include?(value)
-      end
     end
 
     def is_foreign_key_in_current_model?(variable_name)

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.5.0'
+    VERSION = '0.6.0'
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib", "generated_models"]
 
-  s.add_dependency 'ncs_mdes', '~> 0.5'
+  s.add_dependency 'ncs_mdes', '~> 0.6', '>= 0.6.1'
   s.add_dependency 'ncs_navigator_configuration', '~> 0.2'
 
   s.add_dependency 'activesupport', '~> 3.0'

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -471,6 +471,22 @@ module NcsNavigator::Warehouse
         subject.bcdatabase_group.should == :custom
       end
 
+      it 'evaluates constants from NcsNavigator::Warehouse::Transformers' do
+        write_file do |f|
+          f.puts 'c.log_file = "#{EnumTransformer}.log"'
+        end
+
+        subject.log_file.to_s.should == 'NcsNavigator::Warehouse::Transformers::EnumTransformer.log'
+      end
+
+      it 'reports missing constants as bare' do
+        write_file do |f|
+          f.puts 'c.add_transformer = ATransformerIForgotToRequire'
+        end
+
+        lambda { subject }.should raise_error(/uninitialized constant ATransformerIForgotToRequire/)
+      end
+
       describe 'source file tracking' do
         it 'knows what file it came from' do
           write_file { }

data/spec/ncs_navigator/warehouse/models_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+# These tests directly examine the generated models for the current
+# SPEC_MDES_VERSION. For tests of general generation logic, see
+# table_modeler_spec.
+
+describe "Generated models for MDES #{spec_mdes_version}", :use_mdes do
+  let(:models_module) { spec_config.models_module }
+
+  it 'does not generate different models with the same name' do
+    models_module.mdes_order.should == models_module.mdes_order.uniq
+  end
+end

data/spec/ncs_navigator/warehouse/table_modeler_spec.rb

@@ -90,10 +90,16 @@ module NcsNavigator::Warehouse
     it 'preserves non-reversible table names' do
       table.instance_eval { @name = 'generational_tableau_1' }
       subject.load!
-      Spec::ModeledTables::GenerationalTableau1.
+      Spec::ModeledTables::GenerationalTableau_1.
         storage_names[:default].should == 'generational_tableau_1'
     end
 
+    it 'preserves numeric suffixes intact' do
+      table.instance_eval { @name = 'generational_tableau2_3_1' }
+      subject.load!
+      lambda { Spec::ModeledTables::GenerationalTableau2_3_1 }.should_not raise_error(NameError)
+    end
+
     [:default, :mdes_warehouse_working, :mdes_warehouse_reporting].each do |repo|
       it "sets the storage name for the #{repo} repo" do
         subject.load!

data/spec/ncs_navigator/warehouse/transformers/apply_global_values_filter_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers
+  describe ApplyGlobalValuesFilter, :use_mdes do
+    let(:filter) {
+      ApplyGlobalValuesFilter.new(spec_config, :values => { :first_name => 'Fred' })
+    }
+
+    let(:person) { mdes_model(:Person).new }
+    let(:contact) { mdes_model(:Contact).new }
+
+    it 'applies a global value if the record has that variable and it is not set' do
+      filter.call([person])
+      person.first_name.should == 'Fred'
+    end
+
+    it 'does not apply a global value if the record does not have that variable' do
+      lambda { filter.call([contact]) }.should_not raise_error
+    end
+
+    it 'does not apply a global value if the record already has a value for that variable' do
+      person.first_name = 'Leo'
+      filter.call([person])
+      person.first_name.should == 'Leo'
+    end
+
+    describe 'default values' do
+      it 'includes the id of the first PSU as :psu_id' do
+        filter.global_values[:psu_id].should == '20000030'
+      end
+
+      it 'includes the recruitment type as :recruit_type' do
+        filter.global_values[:recruit_type].should == '3'
+      end
+
+      [:psu_id, :recruit_type].each do |defaulted_var|
+        it "can receive an override for #{defaulted_var.inspect} during construction" do
+          ApplyGlobalValuesFilter.new(spec_config, :values => { defaulted_var => 'foo' }).
+            global_values[defaulted_var].should == 'foo'
+        end
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/coded_as_missing_filter_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers
+  describe CodedAsMissingFilter, :use_mdes do
+    def call(records)
+      filter.call(records)
+    end
+
+    def self.it_filters_missing_code(bad_code)
+      it "removes a record whose primary key is #{bad_code.inspect}" do
+        records = [
+          mdes_model(:Person).new(:person_id => bad_code),
+          mdes_model(:Person).new(:person_id => '23')
+        ]
+
+        call(records).collect(&:person_id).should == %w(23)
+      end
+
+      it "clears a non-coded, non-required variable whose entire value is #{bad_code.inspect}" do
+        records = [
+          mdes_model(:StudyCenter).new(:sc_id => '2000029', :comments => bad_code)
+        ]
+
+        call(records).first.comments.should be_nil
+      end
+
+      it "clears a foreign key whose value is #{bad_code.inspect}" do
+        records = [
+          mdes_model(:Person).new(:person_id => '4', :new_address_id => bad_code)
+        ]
+
+        call(records).first.new_address_id.should be_nil
+      end
+
+      it "leaves alone a coded variable whose value is #{bad_code.inspect}" do
+        records = [
+          mdes_model(:Person).new(:person_id => '4', :sex => bad_code)
+        ]
+
+        call(records).first.sex.should == bad_code
+      end
+    end
+
+    describe '.call' do
+      let(:filter) { CodedAsMissingFilter }
+
+      %w(-3 -4 -6 -7).each do |bad_code|
+        it_filters_missing_code(bad_code)
+      end
+    end
+
+    describe '#call' do
+      let(:filter) { CodedAsMissingFilter.new(spec_config) }
+
+      %w(-3 -4 -6 -7).each do |bad_code|
+        it_filters_missing_code(bad_code)
+      end
+
+      describe 'with additional codes' do
+        let(:filter) { CodedAsMissingFilter.new(spec_config, :additional_codes => %w(-47)) }
+
+        %w(-3 -4 -47 -6 -7).each do |bad_code|
+          it_filters_missing_code(bad_code)
+        end
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/database_spec.rb

@@ -192,6 +192,7 @@ module NcsNavigator::Warehouse::Transformers
 
     describe '.create_transformer' do
       let(:enumerator_def) { sample_class }
+      let(:filter) { Object.new.tap { |x| def x.call(y); y; end } }
       subject { enumerator_def.create_transformer(configuration) }
 
       it 'creates an EnumTransformer' do
@@ -203,11 +204,18 @@ module NcsNavigator::Warehouse::Transformers
       end
 
       describe 'with options' do
-        subject { enumerator_def.create_transformer(configuration, :repository => :alpha) }
+        subject {
+          enumerator_def.create_transformer(configuration,
+            :repository => :alpha, :filters => [filter])
+        }
 
         it 'passes the options to the enumerable constructor' do
           subject.enum.repository_name.should == :alpha
         end
+
+        it 'passes the filters to the transformer constructor' do
+          subject.filters.to_a.should == [filter]
+        end
       end
     end
 

data/spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb

@@ -24,9 +24,9 @@ module NcsNavigator::Warehouse::Transformers
         end
       }
       let(:records) { [
-          Sample.new(:id => 1, :name => 'One'),
-          Sample.new(:id => 2, :name => 'Two'),
-          Sample.new(:id => 3, :name => 'Three')
+          Sample.new(:id => 1, :psu_id => '20000030', :name => 'One'),
+          Sample.new(:id => 2, :psu_id => '20000030', :name => 'Two'),
+          Sample.new(:id => 3, :psu_id => '20000030', :name => 'Three')
         ] }
       let(:transform_status) { NcsNavigator::Warehouse::TransformStatus.memory_only('test') }
       subject { EnumTransformer.new(config, records) }
@@ -41,30 +41,6 @@ module NcsNavigator::Warehouse::Transformers
         transform_status.transform_errors.should be_empty
       end
 
-      it 'automatically sets the PSU ID if necessary' do
-        records[1].psu_id = '20000042'
-
-        records.each do |m|
-          m.should_receive(:valid?).and_return(true)
-          m.should_receive(:save).and_return(true)
-        end
-
-        subject.transform(transform_status)
-        records.collect(&:psu_id).should == %w(20000030 20000042 20000030)
-      end
-
-      it 'automatically sets recruit_type if necessary' do
-        records[2].recruit_type = '1'
-
-        records.each do |m|
-          m.should_receive(:valid?).and_return(true)
-          m.should_receive(:save).and_return(true)
-        end
-
-        subject.transform(transform_status)
-        records.collect(&:recruit_type).should == %w(3 3 1)
-      end
-
       describe 'with an invalid instance' do
         before do
           records[0].should_receive(:save).and_return(true)
@@ -86,6 +62,50 @@ module NcsNavigator::Warehouse::Transformers
         end
       end
 
+      describe 'with an instance with an invalid PSU' do
+        before do
+          records[0].should_receive(:save).and_return(true)
+          records[1].should_not_receive(:save)
+          records[2].should_receive(:save).and_return(true)
+
+          records[1].psu_id = '20000041'
+        end
+
+        it 'does not save that instance' do
+          subject.transform(transform_status)
+        end
+
+        it 'saves the other instances' do
+          subject.transform(transform_status)
+        end
+
+        it 'records an error' do
+          subject.transform(transform_status)
+          transform_status.transform_errors.collect(&:record_id).should == ['2']
+        end
+
+        describe 'the recorded error' do
+          let(:error) { transform_status.transform_errors.first }
+
+          before do
+            subject.transform(transform_status)
+          end
+
+          it 'has the correct model class' do
+            error.model_class.should == Sample.to_s
+          end
+
+          it 'has the record ID' do
+            error.record_id.should == '2'
+          end
+
+          it 'has a message' do
+            error.message.should ==
+              'Invalid PSU ID "20000041". The list of valid PSU IDs for this Study Center is ["20000030", "20000042"].'
+          end
+        end
+      end
+
       describe 'with an unsaveable instance' do
         before do
           records[0].should_receive(:save).and_return(true)
@@ -202,6 +222,31 @@ module NcsNavigator::Warehouse::Transformers
           transform_status.transform_errors.first.id.should be_nil
         end
       end
+
+      describe 'with a filter set' do
+        let(:filter_one) { lambda { |recs| recs.each { |r| r.name = 'FILTERED' }; recs } }
+        let(:filter_two) { lambda { |recs| recs.each { |r| r.name.downcase! }; recs } }
+        let(:replacing_filter) { lambda { |recs| [records[1]] } }
+
+        def transformer_with_filters(*filters)
+          EnumTransformer.new(config, records, :filters => filters)
+        end
+
+        it 'applies all the filters' do
+
+          transformer_with_filters(filter_one, filter_two).transform(transform_status)
+
+          records.each { |m| m.name.should == 'filtered' }
+        end
+
+        it 'saves only the results from the filter' do
+          records[0].should_not_receive(:save)
+          records[2].should_not_receive(:save)
+          records[1].should_receive(:save).exactly(3).times
+
+          transformer_with_filters(replacing_filter).transform(transform_status)
+        end
+      end
     end
   end
 end

data/spec/ncs_navigator/warehouse/transformers/filters_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers
+  describe Filters do
+    let(:filter_one)   { StubFilter.new('one') }
+    let(:filter_two)   { StubFilter.new('two') }
+    let(:filter_three) { StubFilter.new('three') }
+
+    describe '#initialize' do
+      it 'sets the #filters attribute' do
+        Filters.new([filter_two, filter_one]).filters.should == [filter_two, filter_one]
+      end
+
+      it 'rejects objects that do not have a call method' do
+        lambda { Filters.new([filter_three, Object.new]) }.
+          should raise_error(/Filter 1 \(Object\) does not have a call method/)
+      end
+    end
+
+    describe '#call' do
+      describe 'with some filters' do
+        let(:filters) { Filters.new([filter_one, filter_two, filter_three]) }
+
+        it 'feeds the result from the each into the next' do
+          filter_one.should_receive(:call).with([:foo]).and_return([:foo, :bar])
+          filter_two.should_receive(:call).with([:foo, :bar]).and_return([:bar])
+          filter_three.should_receive(:call).with([:bar]).and_return([:quux])
+
+          filters.call([:foo]).should == [:quux]
+        end
+
+        it 'treats a nil return as []' do
+          filter_one.should_receive(:call).with([:foo]).and_return(nil)
+          filter_two.should_receive(:call).with([]).and_return([:bar])
+
+          filters.call([:foo])
+        end
+
+        it 'treats false as a value' do
+          filter_one.should_receive(:call).with([false])
+
+          filters.call(false)
+        end
+
+        it 'wraps a single input value in an array' do
+          filter_one.should_receive(:call).with([:foo])
+
+          filters.call(:foo)
+        end
+      end
+
+      describe 'with no filters' do
+        let(:filters) { Filters.new([]) }
+
+        it 'returns the input' do
+          filters.call([:foo]).should == [:foo]
+        end
+
+        it 'normalizes nil to []' do
+          filters.call(nil).should == []
+        end
+
+        it 'wraps a single input value in an array' do
+          filters.call(:bar).should == [:bar]
+        end
+      end
+    end
+
+    describe 'enumerableness' do
+      let(:filters) { Filters.new([filter_one, filter_two, filter_three]) }
+
+      it 'is Enumerable' do
+        Filters.ancestors.should include(::Enumerable)
+      end
+
+      it 'delegates #each to the filter list' do
+        filters.collect(&:name).should == %w(one two three)
+      end
+    end
+
+    class StubFilter
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+      end
+
+      def call(records)
+        records
+      end
+
+      def inspect
+        "#<StubFilter #{name.inspect}>"
+      end
+    end
+  end
+end