ncs_mdes_warehouse 0.0.1

data/spec/ncs_navigator/warehouse/transformers/database_spec.rb

@@ -0,0 +1,194 @@
+require File.expand_path('../../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse::Transformers
+  describe Database do
+    def sample_class(&contents)
+      Class.new do
+        include Database
+
+        class_eval(&contents) if contents
+      end
+    end
+
+    let(:configuration) {
+      NcsNavigator::Warehouse::Configuration.new.tap do |c|
+        c.log_file = tmpdir + 'test.log'
+        c.output_level = :quiet
+      end
+    }
+    let(:options) { {} }
+    let(:instance) { cls.new(configuration, options) }
+    let(:cls) { @cls }
+
+    describe '#repository_name' do
+      it 'defaults to the class name, underscored' do
+        @cls = (FooBaz = sample_class)
+        instance.repository_name.should == :foo_baz
+      end
+
+      it 'can be overridden at the class level' do
+        @cls = sample_class do
+          repository :data_goes_here
+        end
+
+        instance.repository_name.should == :data_goes_here
+      end
+
+      it 'can be overridden at the instance level with the :repository key' do
+        @cls = sample_class
+        options[:repository] = :bazQuux
+        instance.repository_name.should == :bazQuux
+      end
+
+      it 'can be overridden at the instance level with the :repository_name key' do
+        @cls = sample_class
+        options[:repository_name] = :bazQuux
+        instance.repository_name.should == :bazQuux
+      end
+    end
+
+    describe '#connection_parameters' do
+      describe 'from bcdatabase', :modifies_warehouse_state do
+        let(:cls) {
+          sample_class do
+            bcdatabase :name => 'ncs_staff_portal'
+          end
+        }
+
+        it 'uses the values set in the class' do
+          instance.connection_parameters['database'].should == 'ncs_sp'
+        end
+
+        it 'can be overridden at the instance level' do
+          options[:bcdatabase] = { :name => 'ncs_staff_portal_test' }
+          instance.connection_parameters['database'].should == 'ncs_sp_test'
+        end
+
+        it 'defaults the group from the environment' do
+          NcsNavigator::Warehouse.env = 'staging'
+          instance.bcdatabase[:group].should == :ncsdb_staging
+        end
+      end
+    end
+
+    shared_context 'people_pro' do
+      let(:database_file) { 'people_pro.sqlite3' }
+
+      after(:all) do
+        FileUtils.rm_rf database_file
+      end
+    end
+
+    describe '#repository' do
+      include_context 'people_pro'
+
+      let(:cls) do
+        sample_class do
+          repository :people_pro
+
+          bcdatabase :group => 'test_sqlite', :name => 'people_pro'
+        end
+      end
+
+      it 'sets up the repository' do
+        lambda {
+          instance.repository.adapter.options['path'].should == 'people_pro.sqlite3'
+        }.should_not raise_error
+      end
+    end
+
+    describe '#each' do
+      include_context 'people_pro'
+
+      describe 'of .produce_records' do
+        let(:cls) do
+          sample_class do
+            bcdatabase :group => 'test_sqlite', :name => 'people_pro'
+            repository :people_pro
+
+            produce_records :people do |row|
+              row.id
+            end
+
+            produce_records :other_persons, :query => 'SELECT * FROM more_people' do |row|
+              [
+                row.id + 'X',
+                row.id + 'Y'
+              ]
+            end
+          end
+        end
+
+        let(:enumerator_def) { cls }
+        let(:enumerator) { instance }
+
+        before do
+          execute_sql(
+            "CREATE TABLE people (id VARCHAR(24), name VARCHAR(255))",
+            "CREATE TABLE more_people (id VARCHAR(24), name VARCHAR(255))"
+          )
+        end
+
+        after do
+          execute_sql("DROP TABLE people", "DROP TABLE more_people")
+        end
+
+        def execute_sql(*stmts)
+          stmts.each { |stmt| enumerator.repository.adapter.execute stmt }
+        end
+
+        it 'uses the production name as a table name by default' do
+          enumerator_def.record_producers[0].query.should == 'SELECT * FROM people'
+        end
+
+        it 'uses an explicit query if given' do
+          enumerator_def.record_producers[1].query.should == 'SELECT * FROM more_people'
+        end
+
+        it 'can return multiple results per row' do
+          execute_sql(
+            "INSERT INTO people (id) VALUES ('A')",
+            "INSERT INTO more_people (id) VALUES ('S')"
+          )
+
+          enumerator.to_a.should == %w(A SX SY)
+        end
+
+        it 'can execute just one producer' do
+          execute_sql(
+            "INSERT INTO people (id) VALUES ('A')",
+            "INSERT INTO more_people (id) VALUES ('S')"
+          )
+
+          acc = []
+          enumerator.each(:people) do |r|
+            acc << r
+          end
+
+          acc.should == %w(A)
+        end
+      end
+    end
+
+    describe '.create_transformer' do
+      let(:enumerator_def) { sample_class }
+      subject { enumerator_def.create_transformer(configuration) }
+
+      it 'creates an EnumTransformer' do
+        subject.should be_an EnumTransformer
+      end
+
+      it 'creates an enumerator from the source class' do
+        subject.enum.should be_an enumerator_def
+      end
+
+      describe 'with options' do
+        subject { enumerator_def.create_transformer(configuration, :repository => :alpha) }
+
+        it 'passes the options to the enumerable constructor' do
+          subject.enum.repository_name.should == :alpha
+        end
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb

@@ -0,0 +1,94 @@
+require File.expand_path('../../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse::Transformers
+  describe EnumTransformer, :modifies_warehouse_state do
+    class Sample
+      include ::DataMapper::Resource
+
+      property :id, Integer, :key => true
+      property :name, String, :required => true
+    end
+
+    describe '#transform' do
+      let(:records) { [
+          Sample.new(:id => 1, :name => 'One'),
+          Sample.new(:id => 2, :name => 'Two'),
+          Sample.new(:id => 3, :name => 'Three')
+        ] }
+      let(:transform_status) { TransformStatus.new('test') }
+      subject { EnumTransformer.new(records) }
+
+      it 'saves valid records when all are valid' do
+        records.each do |m|
+          m.should_receive(:valid?).and_return(true)
+          m.should_receive(:save).and_return(true)
+        end
+
+        subject.transform(transform_status)
+        transform_status.errors.should be_empty
+      end
+
+      it 'automatically sets the PSU ID if the object accepts it'
+
+      describe 'with an invalid instance' do
+        before do
+          records[0].should_receive(:save).and_return(true)
+          records[1].should_receive(:save).and_return(true)
+          records[2].name = nil
+
+          subject.transform(transform_status)
+        end
+
+        it 'records the invalid instance' do
+          err = transform_status.errors.first
+          err.model_class.should == Sample
+          err.message.should == 'Invalid record. Name must not be blank (name=nil). Sample id=3.'
+        end
+
+        it 'saves the other instances' do
+          # in `before`
+        end
+      end
+
+      describe 'with an unsaveable instance' do
+        before do
+          records[0].should_receive(:save).and_return(true)
+          records[1].should_receive(:save).and_return(false)
+          records[2].should_receive(:save).and_return(true)
+
+          subject.transform(transform_status)
+        end
+
+        it 'records the unsaveable instance' do
+          err = transform_status.errors.first
+          err.model_class.should == Sample
+          err.message.should == 'Could not save. Sample id=2.'
+        end
+
+        it 'saves the saveable instances' do
+          # in `before`
+        end
+      end
+
+      describe 'with an instance that errors on save' do
+        before do
+          records[0].should_receive(:save).and_raise("No database around these parts")
+          records[1].should_receive(:save).and_return(true)
+          records[2].should_receive(:save).and_return(true)
+
+          subject.transform(transform_status)
+        end
+
+        it 'records the failing instance' do
+          err = transform_status.errors.first
+          err.model_class.should == Sample
+          err.message.should == 'Error on save. RuntimeError: No database around these parts. Sample id=1.'
+        end
+
+        it 'saves the saveable instances' do
+          # in `before`
+        end
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<ncs:recruitment_substudy_transmission_envelope xmlns:ncs="http://www.nationalchildrensstudy.gov">
+  <ncs:transmission_header>
+    <sc_id>20000029</sc_id>
+    <psu_id>20000030</psu_id>
+    <specification_version>2.0.01.02</specification_version>
+    <is_snapshot>true</is_snapshot>
+  </ncs:transmission_header>
+  <ncs:transmission_tables>
+    <study_center>
+      <sc_id>20000029</sc_id>
+      <sc_name>Northwestern University</sc_name>
+      <comments>NA</comments>
+      <transaction_type>NA</transaction_type>
+    </study_center>
+    <psu>
+      <sc_id>20000029</sc_id>
+      <psu_id>20000030</psu_id>
+      <psu_name>Cook County, IL</psu_name>
+      <recruit_type>3</recruit_type>
+      <transaction_type>NA</transaction_type>
+    </psu>
+    <ssu>
+      <sc_id>20000029</sc_id>
+      <psu_id>20000030</psu_id>
+      <ssu_id>13</ssu_id>
+      <ssu_name>The unluckiest SSU</ssu_name>
+      <transaction_type>NA</transaction_type>
+    </ssu>
+    <listing_unit>
+      <psu_id>20000030</psu_id>
+      <list_id>100</list_id>
+      <ssu_id>13.1</ssu_id>
+      <tsu_id>-7</tsu_id>
+      <list_line>-7</list_line>
+      <list_source>3</list_source>
+      <list_comment>NA</list_comment>
+      <transaction_type>NA</transaction_type>
+    </listing_unit>
+    <person>
+      <psu_id>20000030</psu_id>
+      <person_id>74</person_id>
+      <prefix>-7</prefix>
+      <first_name></first_name>
+      <last_name></last_name>
+      <middle_name></middle_name>
+      <maiden_name></maiden_name>
+      <suffix>-7</suffix>
+      <title>-7</title>
+      <sex>2</sex>
+      <age>49</age>
+      <age_range>5</age_range>
+      <person_dob>1962-01-01</person_dob>
+      <deceased>2</deceased>
+      <ethnic_group>2</ethnic_group>
+      <person_lang>1</person_lang>
+      <person_lang_oth>-7</person_lang_oth>
+      <maristat>1</maristat>
+      <maristat_oth>-7</maristat_oth>
+      <pref_contact>-4</pref_contact>
+      <pref_contact_oth>-7</pref_contact_oth>
+      <plan_move>2</plan_move>
+      <move_info>-7</move_info>
+      <new_address_id>-3</new_address_id>
+      <when_move>-7</when_move>
+      <date_move>9777-97</date_move>
+      <p_tracing>2</p_tracing>
+      <p_info_source>1</p_info_source>
+      <p_info_source_oth>-7</p_info_source_oth>
+      <p_info_date>2011-03-31</p_info_date>
+      <p_info_update>2011-03-31</p_info_update>
+      <person_comment>NA</person_comment>
+      <transaction_type>NA</transaction_type>
+    </person>
+    <link_contact>
+      <psu_id>20000030</psu_id>
+      <contact_link_id>36</contact_link_id>
+      <contact_id>36</contact_id>
+      <event_id>14</event_id>
+      <instrument_id></instrument_id>
+      <staff_id>staff_portal-frd567</staff_id>
+      <person_id>74</person_id>
+      <provider_id>-7</provider_id>
+      <transaction_type>NA</transaction_type>
+    </link_contact>
+  </ncs:transmission_tables>
+</ncs:recruitment_substudy_transmission_envelope>

data/spec/ncs_navigator/warehouse/transformers/vdr_xml/reader_spec.rb

@@ -0,0 +1,43 @@
+require File.expand_path('../../../../../spec_helper', __FILE__)
+
+class NcsNavigator::Warehouse::Transformers::VdrXml
+  describe Reader, :use_mdes do
+    describe '#each', :slow do
+      let(:filename) { File.expand_path('../made_up_vdr_xml.xml', __FILE__) }
+      let(:reader) { Reader.new(spec_config, filename) }
+      let(:ssu)          { reader.detect { |rec| rec.class.name =~ /Ssu$/ } }
+      let(:person)       { reader.detect { |rec| rec.class.name =~ /Person$/ } }
+      let(:link_contact) { reader.detect { |rec| rec.class.name =~ /LinkContact$/ } }
+
+      it 'verifies the PSU matches the configuration'
+      it 'verifies the specification version matches the runtime MDES version'
+
+      it 'yields all records' do
+        reader.to_a.size.should == 6
+      end
+
+      it 'yields the records using their model classes' do
+        reader.collect { |rec| rec.class.name.demodulize }.should ==
+          %w(StudyCenter Psu Ssu ListingUnit Person LinkContact)
+      end
+
+      it 'reads attribute values' do
+        ssu.ssu_name.should == 'The unluckiest SSU'
+      end
+
+      it 'converts a -3 FK to no assocation' do
+        person.new_address_id.should be_nil
+      end
+
+      describe 'with a blank variable value' do
+        it 'converts a blank FK to no association' do
+          link_contact.instrument_id.should be_nil
+        end
+
+        it 'leaves another blank value as blank' do
+          person.first_name.should == ''
+        end
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/vdr_xml_spec.rb

@@ -0,0 +1,45 @@
+require File.expand_path('../../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse::Transformers
+  describe VdrXml do
+    let(:config) { NcsNavigator::Warehouse::Configuration.new }
+
+    shared_examples 'a VDR transformer' do
+      it 'returns a transformer' do
+        subject.should respond_to(:transform)
+      end
+
+      it 'uses a VDR reader as the enumerable' do
+        subject.enum.should be_a VdrXml::Reader
+      end
+    end
+
+    describe '.from_file' do
+      let(:path) { File.expand_path('../vdr_xml/made_up_vdr_xml.xml', __FILE__) }
+      subject { VdrXml.from_file(config, path) }
+
+      it 'uses the given filename' do
+        subject.enum.filename.should == path
+      end
+
+      include_examples 'a VDR transformer'
+    end
+
+    describe '.from_most_recent_file' do
+      let(:path) { tmpdir('contractor-files') }
+      subject { VdrXml.from_most_recent_file(config, Dir[File.join(path, '*')]) }
+
+      before do
+        system("touch -t 02030405 '#{path}/a'")
+        system("touch -t 02040405 '#{path}/b'")
+        system("touch -t 02020405 '#{path}/c'")
+      end
+
+      it 'uses the most recent filename from the list' do
+        subject.enum.filename.should == File.join(path, 'b')
+      end
+
+      include_examples 'a VDR transformer'
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/xml_emitter_spec.rb

@@ -0,0 +1,108 @@
+require File.expand_path('../../../spec_helper', __FILE__)
+
+require 'zip/zip'
+
+module NcsNavigator::Warehouse
+  describe XmlEmitter, :use_mdes do
+    let(:filename) { tmpdir + 'export.xml' }
+    let(:xml) {
+      XmlEmitter.new(spec_config, filename).emit_xml
+      Nokogiri::XML(File.read(filename))
+    }
+
+    def person_model
+      spec_config.models_module.const_get(:Person)
+    end
+
+    before do
+      spec_config.models_module.mdes_order.each do |model|
+        model.stub!(:all).and_return([])
+      end
+    end
+
+    # Most of the details of the XML are tested on the MdesModel mixin
+    describe 'the generated XML', :slow do
+      it 'includes the SC from the configuration' do
+        xml.xpath('//sc_id').text.should == '2000042'
+      end
+
+      it 'includes the PSU from the configuration' do
+        xml.xpath('//psu_id').text.should == '2000423'
+      end
+
+      it 'includes the appropriate specification version' do
+        xml.xpath('//specification_version').text.
+          should == spec_config.mdes.specification_version
+      end
+
+      describe 'with actual data' do
+        before do
+          person_model.should_receive(:all).and_return([
+              person_model.new(:person_id => 'XQ4'),
+              person_model.new(:person_id => 'QX9')
+            ])
+        end
+
+        it 'contains records for all models' do
+          xml.xpath('//person').size.should == 2
+        end
+
+        it 'contains the right records' do
+          xml.xpath('//person/person_id').collect { |e| e.text.strip }.should == %w(XQ4 QX9)
+        end
+      end
+    end
+
+    describe 'the generated ZIP file', :slow do
+      let(:expected_zipfile) { Pathname.new(filename.to_s + '.zip') }
+
+      before do
+        xml # for side effects
+      end
+
+      it 'exists' do
+        expected_zipfile.should be_readable
+      end
+
+      it 'contains just the XML file' do
+        contents = []
+        Zip::ZipFile.foreach(expected_zipfile) do |entry|
+          contents << entry.name
+        end
+        contents.should == [ filename.basename.to_s ]
+      end
+
+      it 'can be opened by something other than rubyzip' do
+        `which unzip`
+        pending "unzip is not available" unless $? == 0
+
+        `unzip -l '#{expected_zipfile}' 2>&1`.should =~ /#{filename.basename}\s/
+        $?.should == 0
+      end
+    end
+
+    describe 'the default filename', :slow, :use_mdes do
+      subject { XmlEmitter.new(spec_config, nil).filename }
+
+      before do
+        pending "ncs_mdes doesn't work on JRuby" if RUBY_PLATFORM == 'java'
+
+        # Need an actual PSU ID for the default filename code to work
+        NcsNavigator.configuration.psus.first.id = '20000216'
+
+        # Time.parse uses Time.now internally, so this needs to be
+        # defined before starting to register the mock.
+        t = Time.parse('2011-07-28')
+        Time.should_receive(:now).and_return(t)
+      end
+
+      it 'is county_name-DATE.xml' do
+        subject.to_s.should == 'bear_lake-20110728.xml'
+      end
+
+      it 'is a Pathname' do
+        subject.should be_a Pathname
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse_spec.rb

@@ -0,0 +1,35 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+module NcsNavigator
+  describe Warehouse, :modifies_warehouse_state  do
+    describe '.env' do
+      before do
+        clear_warehouse_state
+      end
+
+      it 'defaults to "development"' do
+        Warehouse.env.should == 'development'
+      end
+
+      it 'can be specified in the runtime environment' do
+        ENV['NCS_NAVIGATOR_ENV'] = 'staging'
+        Warehouse.env.should == 'staging'
+      end
+
+      it 'can be set explicitly' do
+        Warehouse.env = 'prod'
+        Warehouse.env.should == 'prod'
+      end
+    end
+
+    describe '.bcdatabase' do
+      before do
+        clear_warehouse_state
+      end
+
+      it 'has a default value' do
+        Warehouse.bcdatabase.should be_a Bcdatabase::DatabaseConfigurations
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,84 @@
+require 'rspec'
+require 'fakefs/spec_helpers'
+
+$LOAD_PATH.push File.expand_path('../../lib', __FILE__)
+
+require 'ncs_navigator/warehouse'
+require 'ncs_navigator/configuration'
+
+$LOAD_PATH.push File.expand_path('..', __FILE__)
+
+require 'global_state_helper'
+
+require 'data_mapper'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+
+  ###### bcdatabase
+
+  config.before(:all) do
+    NcsNavigator::Warehouse.bcdatabase =
+      Bcdatabase.load(File.expand_path('../bcdatabase', __FILE__), :transforms => [:datamapper])
+  end
+
+  ###### MDES model loading
+
+  # Each test run can only operate against one version of the MDES at
+  # a time. When the system supports more than one version, the CI
+  # build should be set up to run with this environment variable set
+  # with each supported version.
+  def spec_mdes_version
+    ENV['SPEC_MDES_VERSION'] || NcsNavigator::Warehouse::DEFAULT_MDES_VERSION
+  end
+
+  ##
+  # A mostly-default configuration to use in tests of objects that
+  # rely on the MDES specification so as to avoid initializing it
+  # multiple times.
+  def spec_config
+    @spec_config || fail("Flag specs that use spec_config with :use_mdes")
+  end
+
+  config.before(:each, :use_mdes) do
+    @spec_config ||= NcsNavigator::Warehouse::Configuration.new.tap do |c|
+      c.mdes_version = spec_mdes_version
+      c.output_level = :quiet
+      c.log_file = tmpdir + 'spec.log'
+    end
+  end
+
+  ###### modifies_warehouse_state
+
+  config.before(:each, :modifies_warehouse_state) do
+    @global_state_preserver = NcsNavigator::Warehouse::Spec::GlobalStatePreserver.new.save
+  end
+
+  def clear_warehouse_state
+    @global_state_preserver.clear
+  end
+
+  config.after(:each, :modifies_warehouse_state) do
+    @global_state_preserver.restore
+  end
+
+  ###### tmpdir
+
+  config.after do
+    @tmpdir.rmtree if @tmpdir && !ENV['KEEP_TMP']
+  end
+
+  # @return [Pathname] the path to an existing temporary director
+  def tmpdir(*path)
+    @tmpdir ||= Pathname.new(File.expand_path('../../tmp', __FILE__)).
+      tap { |p| p.mkpath }
+    if path.empty?
+      @tmpdir
+    else
+      @tmpdir.join(*path).tap { |p| p.mkpath }
+    end
+  end
+end
+
+NcsNavigator.configuration =
+  NcsNavigator::Configuration.new(File.expand_path('../navigator.ini', __FILE__))

data/spec/test_samples.csv

@@ -0,0 +1,2 @@
+PSU_ID, AREA, SSU_ID, SSU_NAME, TSU_ID, TSU_NAME
+2000423, Uptown, 42, UPT-42, 42-3, UPT-42X