ncs_mdes_warehouse 0.0.2 → 0.1.0

data/spec/ncs_navigator/warehouse/transformers/database_spec.rb

@@ -116,6 +116,9 @@ module NcsNavigator::Warehouse::Transformers
                 row.id + 'Y'
               ]
             end
+
+            produce_records :nothing, :query => 'SELECT * FROM people' do |row|
+            end
           end
         end
 
@@ -176,6 +179,14 @@ module NcsNavigator::Warehouse::Transformers
 
           enumerator.to_a(:people).should == %w(B)
         end
+
+        it 'yields nothing for an producer that returns nil' do
+          execute_sql(
+            "INSERT INTO people (id) VALUES ('A')"
+          )
+
+          enumerator.to_a(:nothing).should == []
+        end
       end
     end
 
@@ -211,7 +222,7 @@ module NcsNavigator::Warehouse::Transformers
         end
       end
 
-      describe '#model_row' do
+      describe '#produce_one_for_one' do
         # DataMapper models can't be anonymous
         module Database::DSL::TestModels
           class Address
@@ -234,6 +245,7 @@ module NcsNavigator::Warehouse::Transformers
 
         let(:options) { {} }
         let(:cls) { sample_class }
+        let(:producer) { cls.record_producers.first }
 
         def make_row(contents)
           Struct.new(*contents.keys).new.tap do |r|
@@ -244,7 +256,8 @@ module NcsNavigator::Warehouse::Transformers
         end
 
         def model_row(row)
-          cls.model_row(address_model, make_row(row), options)
+          cls.produce_one_for_one(:addresses, address_model, options)
+          producer.row_processor.call(make_row(row))
         end
 
         it 'maps a column to the property with the same name' do
@@ -271,13 +284,13 @@ module NcsNavigator::Warehouse::Transformers
               model_row(:address_type => '-5', :address_length => '6')
               fail "Exception not thrown"
             rescue Database::UnusedColumnsForModelError => e
-              e.unused.should == [:address_length]
+              e.unused.should == ['address_length']
             end
           end
 
           it 'does not fail if the unused column is ignored in the options' do
             options[:on_unused] = :fail
-            options[:ignored_or_used] = %w(address_length)
+            options[:ignored_columns] = %w(address_length)
             lambda { model_row(:address_type => '-5', :address_length => '6') }.
               should_not raise_error
           end
@@ -301,7 +314,7 @@ module NcsNavigator::Warehouse::Transformers
                 model_row(:address_type => '-5', :address_length => '6')
                 fail "Exception not thrown"
               rescue Database::UnusedColumnsForModelError => e
-                e.unused.should == [:address_length]
+                e.unused.should == %w(address_length)
               end
             end
 
@@ -346,31 +359,14 @@ module NcsNavigator::Warehouse::Transformers
             ).street.should == '123 Anymain Dr.'
           end
 
-          it 'prefers the mapped column value to an explicit property value' do
-            options[:property_values] = { :street => '456 Anywhere St.' }
-            model_row(:street_loc => '123 Anymain Dr.').street.should == '123 Anymain Dr.'
-          end
-        end
-
-        describe 'with property value mappings' do
-          before do
-            options[:property_values] = {
-              :street => '456 Anywhere St.'
-            }
-          end
-
-          it 'prefers the explicit mapping to a column' do
-            model_row(:street => '123 Anymain Dr.').street.should == '456 Anywhere St.'
+          it 'does not include the default mapping in the total map' do
+            cls.produce_one_for_one(:addresses, address_model, options)
+            producer.column_map(%w(street street_loc)).keys.should == %w(street_loc)
           end
 
-          it 'reports the column as unused' do
-            options[:on_unused] = :fail
-            begin
-              model_row(:street => '123 Anymain Dr.').street.should == '456 Anywhere St.'
-              fail "Exception not thrown"
-            rescue Database::UnusedColumnsForModelError => e
-              e.unused.should include(:street)
-            end
+          it 'always includes explicit mapping values in the total map' do
+            cls.produce_one_for_one(:addresses, address_model, options)
+            producer.column_map(%w(street)).keys.should == %w(street_loc)
           end
         end
       end

data/spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb

@@ -9,14 +9,19 @@ module NcsNavigator::Warehouse::Transformers
       property :name, String, :required => true
     end
 
+    before(:all) do
+      DataMapper.finalize
+    end
+
     describe '#transform' do
+      let(:config) { NcsNavigator::Warehouse::Configuration.new }
       let(:records) { [
           Sample.new(:id => 1, :name => 'One'),
           Sample.new(:id => 2, :name => 'Two'),
           Sample.new(:id => 3, :name => 'Three')
         ] }
-      let(:transform_status) { TransformStatus.new('test') }
-      subject { EnumTransformer.new(records) }
+      let(:transform_status) { NcsNavigator::Warehouse::TransformStatus.memory_only('test') }
+      subject { EnumTransformer.new(config, records) }
 
       it 'saves valid records when all are valid' do
         records.each do |m|
@@ -25,7 +30,7 @@ module NcsNavigator::Warehouse::Transformers
         end
 
         subject.transform(transform_status)
-        transform_status.errors.should be_empty
+        transform_status.transform_errors.should be_empty
       end
 
       it 'automatically sets the PSU ID if the object accepts it'
@@ -40,8 +45,8 @@ module NcsNavigator::Warehouse::Transformers
         end
 
         it 'records the invalid instance' do
-          err = transform_status.errors.first
-          err.model_class.should == Sample
+          err = transform_status.transform_errors.first
+          err.model_class.should == Sample.to_s
           err.message.should == 'Invalid record. Name must not be blank (name=nil). Sample id=3.'
         end
 
@@ -60,8 +65,8 @@ module NcsNavigator::Warehouse::Transformers
         end
 
         it 'records the unsaveable instance' do
-          err = transform_status.errors.first
-          err.model_class.should == Sample
+          err = transform_status.transform_errors.first
+          err.model_class.should == Sample.to_s
           err.message.should == 'Could not save. Sample id=2.'
         end
 
@@ -80,9 +85,10 @@ module NcsNavigator::Warehouse::Transformers
         end
 
         it 'records the failing instance' do
-          err = transform_status.errors.first
-          err.model_class.should == Sample
-          err.message.should == 'Error on save. RuntimeError: No database around these parts. Sample id=1.'
+          err = transform_status.transform_errors.first
+          err.model_class.should == Sample.to_s
+          err.message.should ==
+            'Error on save. RuntimeError: No database around these parts. Sample id=1.'
         end
 
         it 'saves the saveable instances' do

data/spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<ncs:recruitment_substudy_transmission_envelope xmlns:ncs="http://www.nationalchildrensstudy.gov">
+<ncs:recruitment_substudy_transmission_envelope xmlns:ncs="http://www.nationalchildrensstudy.gov" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <ncs:transmission_header>
     <sc_id>20000029</sc_id>
     <psu_id>20000030</psu_id>
@@ -32,7 +32,7 @@
       <list_id>100</list_id>
       <ssu_id>13.1</ssu_id>
       <tsu_id>-7</tsu_id>
-      <list_line>-7</list_line>
+      <list_line random_attr='foo'>-7</list_line>
       <list_source>3</list_source>
       <list_comment>NA</list_comment>
       <transaction_type>NA</transaction_type>
@@ -41,7 +41,7 @@
       <psu_id>20000030</psu_id>
       <person_id>74</person_id>
       <prefix>-7</prefix>
-      <first_name></first_name>
+      <first_name>Josephine</first_name>
       <last_name></last_name>
       <middle_name></middle_name>
       <maiden_name></maiden_name>
@@ -70,7 +70,7 @@
       <p_info_date>2011-03-31</p_info_date>
       <p_info_update>2011-03-31</p_info_update>
       <person_comment>NA</person_comment>
-      <transaction_type>NA</transaction_type>
+      <transaction_type xsi:nil="true"/>
     </person>
     <link_contact>
       <psu_id>20000030</psu_id>

data/spec/ncs_navigator/warehouse/transformers/vdr_xml/reader_spec.rb

@@ -8,6 +8,7 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
       let(:ssu)          { reader.detect { |rec| rec.class.name =~ /Ssu$/ } }
       let(:person)       { reader.detect { |rec| rec.class.name =~ /Person$/ } }
       let(:link_contact) { reader.detect { |rec| rec.class.name =~ /LinkContact$/ } }
+      let(:listing_unit) { reader.detect { |rec| rec.class.name =~ /ListingUnit$/ } }
 
       it 'verifies the PSU matches the configuration'
       it 'verifies the specification version matches the runtime MDES version'
@@ -21,21 +22,25 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
           %w(StudyCenter Psu Ssu ListingUnit Person LinkContact)
       end
 
-      it 'reads attribute values' do
-        ssu.ssu_name.should == 'The unluckiest SSU'
+      it 'reads variable values' do
+        person.first_name.should == 'Josephine'
       end
 
       it 'converts a -3 FK to no assocation' do
         person.new_address_id.should be_nil
       end
 
+      it 'ignores random attributes' do
+        listing_unit.list_line.should == "-7"
+      end
+
       describe 'with a blank variable value' do
         it 'converts a blank FK to no association' do
           link_contact.instrument_id.should be_nil
         end
 
         it 'leaves another blank value as blank' do
-          person.first_name.should == ''
+          person.last_name.should == ''
         end
       end
     end

data/spec/spec_helper.rb

@@ -70,7 +70,7 @@ RSpec.configure do |config|
 
   # @return [Pathname] the path to an existing temporary director
   def tmpdir(*path)
-    @tmpdir ||= Pathname.new(File.expand_path('../../tmp', __FILE__)).
+    @tmpdir = Pathname.new(File.expand_path('../../tmp', __FILE__)).
       tap { |p| p.mkpath }
     if path.empty?
       @tmpdir

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ncs_mdes_warehouse
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-04 00:00:00.000000000Z
+date: 2011-11-14 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ncs_mdes
-  requirement: &2159237800 !ruby/object:Gem::Requirement
+  requirement: &2154732560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -24,10 +24,10 @@ dependencies:
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *2159237800
+  version_requirements: *2154732560
 - !ruby/object:Gem::Dependency
   name: ncs_navigator_configuration
-  requirement: &2159236980 !ruby/object:Gem::Requirement
+  requirement: &2154731640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -35,10 +35,10 @@ dependencies:
         version: '0.1'
   type: :runtime
   prerelease: false
-  version_requirements: *2159236980
+  version_requirements: *2154731640
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2159236440 !ruby/object:Gem::Requirement
+  requirement: &2154731040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -46,10 +46,10 @@ dependencies:
         version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2159236440
+  version_requirements: *2154731040
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &2159235940 !ruby/object:Gem::Requirement
+  requirement: &2154730440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -57,10 +57,10 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *2159235940
+  version_requirements: *2154730440
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &2159235400 !ruby/object:Gem::Requirement
+  requirement: &2154729880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -68,10 +68,10 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *2159235400
+  version_requirements: *2154729880
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &2159234800 !ruby/object:Gem::Requirement
+  requirement: &2154729320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -79,10 +79,10 @@ dependencies:
         version: 0.9.4
   type: :runtime
   prerelease: false
-  version_requirements: *2159234800
+  version_requirements: *2154729320
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2159234100 !ruby/object:Gem::Requirement
+  requirement: &2154728700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -90,10 +90,10 @@ dependencies:
         version: 1.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *2159234100
+  version_requirements: *2154728700
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &2159233480 !ruby/object:Gem::Requirement
+  requirement: &2154728100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -101,10 +101,10 @@ dependencies:
         version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2159233480
+  version_requirements: *2154728100
 - !ruby/object:Gem::Dependency
   name: data_mapper
-  requirement: &2159232820 !ruby/object:Gem::Requirement
+  requirement: &2154727600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -112,10 +112,10 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2159232820
+  version_requirements: *2154727600
 - !ruby/object:Gem::Dependency
   name: bcdatabase
-  requirement: &2159232220 !ruby/object:Gem::Requirement
+  requirement: &2154726800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -123,10 +123,10 @@ dependencies:
         version: '1.1'
   type: :runtime
   prerelease: false
-  version_requirements: *2159232220
+  version_requirements: *2154726800
 - !ruby/object:Gem::Dependency
   name: dm-postgres-adapter
-  requirement: &2159231600 !ruby/object:Gem::Requirement
+  requirement: &2154709640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -134,10 +134,10 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2159231600
+  version_requirements: *2154709640
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2159230940 !ruby/object:Gem::Requirement
+  requirement: &2154709020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -145,10 +145,10 @@ dependencies:
         version: '2.6'
   type: :development
   prerelease: false
-  version_requirements: *2159230940
+  version_requirements: *2154709020
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2159230280 !ruby/object:Gem::Requirement
+  requirement: &2154708560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -156,10 +156,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2159230280
+  version_requirements: *2154708560
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &2159225600 !ruby/object:Gem::Requirement
+  requirement: &2154707960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -167,10 +167,10 @@ dependencies:
         version: 0.7.2
   type: :development
   prerelease: false
-  version_requirements: *2159225600
+  version_requirements: *2154707960
 - !ruby/object:Gem::Dependency
   name: ci_reporter
-  requirement: &2159225100 !ruby/object:Gem::Requirement
+  requirement: &2154707300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -178,10 +178,10 @@ dependencies:
         version: 1.6.5
   type: :development
   prerelease: false
-  version_requirements: *2159225100
+  version_requirements: *2154707300
 - !ruby/object:Gem::Dependency
   name: fakefs
-  requirement: &2159224500 !ruby/object:Gem::Requirement
+  requirement: &2154706660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -189,7 +189,7 @@ dependencies:
         version: 0.4.0
   type: :development
   prerelease: false
-  version_requirements: *2159224500
+  version_requirements: *2154706660
 description: 
 email:
 - r-sutphin@northwestern.edu
@@ -483,13 +483,16 @@ files:
 - lib/ncs_navigator/warehouse/models.rb
 - lib/ncs_navigator/warehouse/models/mdes_model.rb
 - lib/ncs_navigator/warehouse/models/mdes_model_collection.rb
+- lib/ncs_navigator/warehouse/postgresql.rb
+- lib/ncs_navigator/warehouse/postgresql/pgpass.rb
 - lib/ncs_navigator/warehouse/table_modeler.rb
 - lib/ncs_navigator/warehouse/table_modeler/mdes_ext.rb
 - lib/ncs_navigator/warehouse/table_modeler/model_template.rb.erb
+- lib/ncs_navigator/warehouse/transform_load.rb
+- lib/ncs_navigator/warehouse/transform_status.rb
 - lib/ncs_navigator/warehouse/transformers.rb
 - lib/ncs_navigator/warehouse/transformers/database.rb
 - lib/ncs_navigator/warehouse/transformers/enum_transformer.rb
-- lib/ncs_navigator/warehouse/transformers/transform_status.rb
 - lib/ncs_navigator/warehouse/transformers/vdr_xml.rb
 - lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb
 - lib/ncs_navigator/warehouse/updating_shell.rb
@@ -504,7 +507,9 @@ files:
 - spec/navigator.ini
 - spec/ncs_navigator/warehouse/configuration_spec.rb
 - spec/ncs_navigator/warehouse/models/mdes_model_spec.rb
+- spec/ncs_navigator/warehouse/postgresql/pgpass_spec.rb
 - spec/ncs_navigator/warehouse/table_modeler_spec.rb
+- spec/ncs_navigator/warehouse/transform_load_spec.rb
 - spec/ncs_navigator/warehouse/transformers/database_spec.rb
 - spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml
@@ -529,7 +534,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3545445795691480899
+      hash: -4006681032861892905
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -538,7 +543,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3545445795691480899
+      hash: -4006681032861892905
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10
@@ -554,7 +559,9 @@ test_files:
 - spec/navigator.ini
 - spec/ncs_navigator/warehouse/configuration_spec.rb
 - spec/ncs_navigator/warehouse/models/mdes_model_spec.rb
+- spec/ncs_navigator/warehouse/postgresql/pgpass_spec.rb
 - spec/ncs_navigator/warehouse/table_modeler_spec.rb
+- spec/ncs_navigator/warehouse/transform_load_spec.rb
 - spec/ncs_navigator/warehouse/transformers/database_spec.rb
 - spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml