ncs_mdes_warehouse 0.6.1 → 0.6.2

data/.gitignore

@@ -6,3 +6,4 @@ reports
 *.sqlite3
 logs
 pkg
+*.log

data/CHANGELOG.md

@@ -1,6 +1,15 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.6.2
+-----
+
+- Added `Configuration#model` to find a model class for a particular
+  table name or unqualified class name in the current MDES version.
+
+- Added transformer for automatically setting event start dates from
+  the earliest associated contact (if any). (#2194)
+
 0.6.1
 -----
 

data/Rakefile

@@ -21,6 +21,20 @@ namespace :spec do
   RSpec::Core::RakeTask.new(:all) do |t|
     t.pattern = ENV['SPEC_PATTERN'] || "spec/**/*_spec.rb"
   end
+
+  task :load_path do
+    $LOAD_PATH.unshift File.expand_path('../spec', __FILE__)
+  end
+
+  desc 'Creates the database for the specs'
+  task :db => :load_path do
+    require 'spec_warehouse_config'
+    NcsNavigator::Warehouse::Spec.database_initializer.tap do |init|
+      init.set_up_repository(:both)
+      init.replace_schema
+      init.clone_working_to_reporting
+    end
+  end
 end
 
 namespace :generate do
@@ -40,5 +54,5 @@ namespace :ci do
     ENV['SPEC_OPTS'] = "#{ENV['SPEC_OPTS']} --format nested --no-color"
   end
 
-  task :spec => [:spec_setup, 'ci:setup:rspecbase', 'rake:spec']
+  task :spec => [:spec_setup, 'ci:setup:rspecbase', 'rake:spec:db', 'rake:spec']
 end

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -161,6 +161,22 @@ module NcsNavigator::Warehouse
     end
     private :set_default_mdes_version
 
+    ##
+    # @param [String,Symbol] table_or_model_name either an MDES table
+    #   name or the unqualified name of an MDES Warehouse model
+    #
+    # @return [Class,nil] the model in the current MDES version
+    #   corresponding to the given model or table name, or nil if
+    #   there is no such model
+    def model(table_or_model_name)
+      begin
+        models_module.const_get(table_or_model_name)
+      rescue NameError
+        models_module.mdes_order.
+          find { |model| model.mdes_table_name.to_s == table_or_model_name.to_s }
+      end
+    end
+
     ####
     #### Suite configuration
     ####

data/lib/ncs_navigator/warehouse/database_initializer.rb

@@ -3,6 +3,7 @@ require 'ncs_navigator/warehouse'
 require 'bcdatabase'
 require 'benchmark'
 require 'forwardable'
+require 'tempfile'
 
 require 'ncs_navigator/warehouse/data_mapper'
 require 'ncs_navigator/warehouse/data_mapper_patches'

data/lib/ncs_navigator/warehouse/transformers.rb

@@ -15,6 +15,7 @@ module NcsNavigator::Warehouse
     autoload :CodedAsMissingFilter,    'ncs_navigator/warehouse/transformers/coded_as_missing_filter'
     autoload :Database,                'ncs_navigator/warehouse/transformers/database'
     autoload :EnumTransformer,         'ncs_navigator/warehouse/transformers/enum_transformer'
+    autoload :EventStartFromContactTransformer, 'ncs_navigator/warehouse/transformers/event_start_from_contact_transformer'
     autoload :Filters,                 'ncs_navigator/warehouse/transformers/filters'
     autoload :NoBlankForeignKeysFilter,       'ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter'
     autoload :NoSsuOutreachAllSsusFilter,     'ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter'

data/lib/ncs_navigator/warehouse/transformers/event_start_from_contact_transformer.rb

@@ -0,0 +1,98 @@
+require 'ncs_navigator/warehouse'
+
+require 'forwardable'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # This transformer finds all events without start dates and attempts
+  # to set them from the earliest associated contact.
+  class EventStartFromContactTransformer
+    extend Forwardable
+    attr_reader :configuration
+
+    def_delegators :@configuration, :log, :shell
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    ##
+    # Implements this transformer's behavior.
+    #
+    # @param [TransformStatus] status
+    # @return [void]
+    def transform(status)
+      configuration.model(:Event).all(:event_start_date.like => '9%').each do |event|
+        candidates = ::DataMapper.repository.adapter.select(%Q{
+          SELECT c.contact_date, c.contact_start_time
+          FROM link_contact lc INNER JOIN contact c ON lc.contact_id=c.contact_id
+          WHERE lc.event_id='#{event.event_id}'
+          ORDER BY c.contact_date, c.contact_start_time
+        })
+
+        start_date = candidates.
+          select { |dt| dt.contact_date !~ /^9/ }.collect { |dt| dt.contact_date }.uniq.first
+        start_time = candidates.
+          select { |dt| dt.contact_date == start_date && dt.contact_start_time !~ /^9/ }.
+          collect { |dt| dt.contact_start_time }.first
+
+        if start_date || start_time
+          event.event_start_date = start_date if start_date
+          event.event_start_time = start_time if start_time
+
+          save(event, status)
+
+          status.record_count += 1
+        end
+      end
+    end
+
+    private
+
+    # TODO: share this stuff between here and EnumTransformer
+
+    def save(record, status)
+      if record.valid?
+        log.debug("Saving valid transformed event record #{record_ident record}.")
+        begin
+          unless record.save
+            msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
+            log.error msg
+            status.unsuccessful_record(record, msg)
+          end
+        rescue => e
+          msg = "Error on save. #{e.class}: #{e}."
+          log.error msg
+          status.unsuccessful_record(record, msg)
+        end
+      else
+        log.error "Event invalid after transformation. #{record_messages(record).join(' ')}"
+        record.errors.keys.each do |prop|
+          record.errors[prop].each do |e|
+            status.unsuccessful_record(
+              record, "Invalid after transformation: #{e}.",
+              :attribute_name => prop,
+              :attribute_value => record.send(prop).inspect
+              )
+          end
+        end
+      end
+    end
+
+    def record_messages(record)
+      record.errors.keys.collect { |prop|
+        record.errors[prop].collect { |e|
+          v = record.send(prop)
+          "#{e} (#{prop}=#{v.inspect})."
+        }
+      }.flatten
+    end
+
+    def record_ident(rec)
+      # No composite keys in the MDES
+      '%s %s=%s' % [
+        rec.class.name.demodulize, rec.class.key.first.name, rec.key.try(:first).inspect]
+    end
+  end
+end
+

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.6.1'
+    VERSION = '0.6.2'
   end
 end

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -127,6 +127,50 @@ module NcsNavigator::Warehouse
       # see above for test of non-default value
     end
 
+    describe '#model' do
+      context 'for a known version', :slow, :use_mdes, :modifies_warehouse_state do
+        describe 'by table name' do
+          let(:actual) { config.model('link_contact') }
+
+          it 'finds the model' do
+            actual.to_s.should =~ /LinkContact$/
+          end
+
+          it 'returns a class' do
+            actual.should be_a Class
+          end
+
+          it 'returns a class in the models module' do
+            actual.to_s.should =~ %r[^#{config.models_module.to_s}]
+          end
+
+          it 'returns nil if no match' do
+            config.model('quux').should be_nil
+          end
+        end
+
+        describe 'by model name' do
+          let(:actual) { config.model(:Person) }
+
+          it 'finds the model' do
+            actual.to_s.should =~ /Person$/
+          end
+
+          it 'returns a class' do
+            actual.should be_a Class
+          end
+
+          it 'returns a class in the models module' do
+            actual.to_s.should =~ %r[^#{config.models_module.to_s}]
+          end
+
+          it 'returns nil if no match' do
+            config.model(:Foo).should be_nil
+          end
+        end
+      end
+    end
+
     describe '#navigator' do
       it 'defaults to the global default instance' do
         config.navigator.should be(NcsNavigator.configuration)

data/spec/ncs_navigator/warehouse/transformers/event_start_from_contact_transformer_spec.rb

@@ -0,0 +1,94 @@
+require 'spec_helper'
+require 'warehouse_record_creation_helpers'
+
+module NcsNavigator::Warehouse::Transformers
+  describe EventStartFromContactTransformer, :use_database, :slow do
+    include_context :warehouse_record_creation_helpers
+
+    let(:transformer) { EventStartFromContactTransformer.new(spec_config) }
+    let(:transform_status) { NcsNavigator::Warehouse::TransformStatus.memory_only('test') }
+
+    def create_contact_for_event(event, contact_attributes)
+      contact = create_warehouse_record_with_defaults(:Contact, contact_attributes)
+      create_warehouse_record_with_defaults(:LinkContact,
+        :contact_link_id => "LC_#{event.event_id}_#{contact.contact_id}",
+        :event => event, :contact => contact, :staff => staff
+        )
+      contact
+    end
+
+    let(:staff) { create_warehouse_record_with_defaults(:Staff, :staff_id => 'foo') }
+
+    let!(:ok_event) {
+      create_warehouse_record_with_defaults(:Event,
+        :event_id => 'E_OK', :event_start_date => '2010-09-04')
+    }
+    let!(:ok_event_contact) {
+      create_contact_for_event(ok_event,
+        :contact_id => 'C_OK', :contact_date => '2010-09-01')
+    }
+
+    let!(:bare_event) {
+      create_warehouse_record_with_defaults(:Event,
+        :event_id => 'E_BARE', :event_start_date => '9666-96-96'
+      )
+    }
+
+    let!(:bad_event) {
+      create_warehouse_record_with_defaults(:Event,
+        :event_id => 'E_BAD', :event_start_date => '9666-96-96')
+    }
+    let!(:bad_event_contact_0) {
+      create_contact_for_event(bad_event,
+        :contact_id => 'C_BAD_0', :contact_date => '2010-02-07', :contact_start_time => '00:00')
+    }
+    let!(:bad_event_contact_1) {
+      create_contact_for_event(bad_event,
+        :contact_id => 'C_BAD_1', :contact_date => '2010-02-05', :contact_start_time => '96:96')
+    }
+    let!(:bad_event_contact_2) {
+      create_contact_for_event(bad_event,
+        :contact_id => 'C_BAD_2', :contact_date => '2010-02-05', :contact_start_time => '03:21')
+    }
+    let!(:bad_event_contact_3) {
+      create_contact_for_event(bad_event,
+        :contact_id => 'C_BAD_3', :contact_date => '2010-02-05', :contact_start_time => '02:34')
+    }
+    let!(:bad_event_contact_4) {
+      create_contact_for_event(bad_event,
+        :contact_id => 'C_BAD_4', :contact_date => '9666-97-97', :contact_start_time => '01:10')
+    }
+
+    def reload(record)
+      record.model.first(record.model.key.first => record.key.first)
+    end
+
+    before do
+      transformer.transform(transform_status)
+    end
+
+    it 'leaves events with start dates alone' do
+      reload(ok_event).event_start_date.should == '2010-09-04'
+    end
+
+    it 'leaves events without contacts alone' do
+      reload(bare_event).event_start_date.should == '9666-96-96'
+    end
+
+    describe 'for an event without a start date' do
+      let(:fixed_event) { reload(bad_event) }
+
+      it "takes the event start date from the earliest associated contact's start date" do
+        fixed_event.event_start_date.should == '2010-02-05'
+      end
+
+      it "takes the event start time from the earliest associated contact's start time" do
+        fixed_event.event_start_time.should == '02:34'
+      end
+
+      it 'records the number events changed' do
+        transform_status.record_count.should == 1
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -9,6 +9,7 @@ require 'ncs_navigator/configuration'
 $LOAD_PATH.push File.expand_path('..', __FILE__)
 
 require 'global_state_helper'
+require 'spec_warehouse_config'
 
 require 'dm-core'
 
@@ -21,17 +22,11 @@ RSpec.configure do |config|
   # multiple times. It also contains the configuration pointing to the
   # test database schemas.
   def spec_config
-    @spec_config || fail("Flag specs that use spec_config with :use_mdes or :use_database")
-  end
+    require 'spec_warehouse_config'
 
-  def init_spec_config
-    @spec_config ||= NcsNavigator::Warehouse::Configuration.new.tap do |c|
-      c.mdes_version = spec_mdes_version
+    @spec_config ||= NcsNavigator::Warehouse::Spec.configuration.tap do |c|
       c.output_level = :quiet
       c.log_file = tmpdir + 'spec.log'
-      c.bcdatabase_group = ENV['CI_RUBY'] ? :public_ci_postgresql9 : :local_postgresql
-      c.bcdatabase_entries[:working] = :mdes_warehouse_working_test
-      c.bcdatabase_entries[:reporting] = :mdes_warehouse_reporting_test
     end
   end
 
@@ -41,26 +36,18 @@ RSpec.configure do |config|
   # @param name [Symbol] an MDES table name or a warehouse model name
   # @return [Class]
   def mdes_model(name)
-    begin
-      spec_config.models_module.const_get(name)
-    rescue NameError
-      spec_config.models_module.mdes_order.find { |model| model.mdes_table_name.to_s == name.to_s }
-    end
+    spec_config.model(name)
   end
 
   ###### MDES model loading
 
-  # Each test run can only operate against one version of the MDES at
-  # a time. The CI build is set up to run serially with this
-  # environment variable set with each supported version.
   def spec_mdes_version
-    @spec_mdes_version ||=
-      (ENV['SPEC_MDES_VERSION'] || NcsNavigator::Warehouse::DEFAULT_MDES_VERSION).
-      gsub(/[^\d\.]/, '')
+    NcsNavigator::Warehouse::Spec.mdes_version
   end
 
   config.before(:each, :use_mdes) do
-    init_spec_config
+    # Specs with :use_mdes may expect the warehouse models to be loaded
+    spec_config
   end
 
   ###### modifies_warehouse_state
@@ -80,24 +67,28 @@ RSpec.configure do |config|
   ###### use_database
 
   config.before(:each, :use_database) do
-    init_spec_config
-
     $db_init ||=
       begin
         init = NcsNavigator::Warehouse::DatabaseInitializer.new(spec_config)
         init.set_up_repository(:both)
-        init.replace_schema
-        init.clone_working_to_reporting
       end
   end
 
   config.after(:each, :use_database) do
-    DataMapper.repository.adapter.execute("SET client_min_messages = warning")
-    ::DataMapper::Model.descendants.each do |model|
-      begin
-        DataMapper.repository.adapter.execute("TRUNCATE TABLE #{model.storage_name} CASCADE")
-      rescue DataObjects::SyntaxError
-        # table was never created
+    [:working, :reporting].each do |db|
+      repo_name = "mdes_warehouse_#{db}".to_sym
+      adapter = DataMapper.repository(repo_name).adapter
+      adapter.execute("SET client_min_messages = warning")
+
+      tables = adapter.
+        select("SELECT table_name FROM information_schema.tables WHERE table_schema='public'")
+      unless tables.empty?
+        begin
+          DataMapper.repository.adapter.execute("TRUNCATE TABLE #{tables.join(', ')} CASCADE")
+        rescue DataObjects::SyntaxError => e
+          # some table was never created
+          $stderr.puts "Post-spec truncation failed: #{e}. This may not indicate a problem; just letting you know."
+        end
       end
     end
   end

data/spec/spec_warehouse_config.rb

@@ -0,0 +1,30 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse
+  module Spec
+    # Each test run can only operate against one version of the MDES at
+    # a time. The CI build is set up to run serially with this
+    # environment variable set with each supported version.
+    def self.mdes_version
+      @mdes_version ||=
+        (ENV['SPEC_MDES_VERSION'] || NcsNavigator::Warehouse::DEFAULT_MDES_VERSION).
+        gsub(/[^\d\.]/, '')
+    end
+
+    def self.configuration
+      @configuration ||= NcsNavigator::Warehouse::Configuration.new.tap do |c|
+        c.mdes_version = self.mdes_version
+        # This is used for rake invocations only; within specs the log
+        # goes to the spec tmpdir for cleanup.
+        c.log_file = File.expand_path('../spec.log', __FILE__)
+        c.bcdatabase_group = ENV['CI_RUBY'] ? :public_ci_postgresql9 : :local_postgresql
+        c.bcdatabase_entries[:working] = :mdes_warehouse_working_test
+        c.bcdatabase_entries[:reporting] = :mdes_warehouse_reporting_test
+      end
+    end
+
+    def self.database_initializer
+      @database_initializer ||= DatabaseInitializer.new(configuration)
+    end
+  end
+end

data/spec/warehouse_record_creation_helpers.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+shared_context :warehouse_record_creation_helpers do
+  def save_wh(record)
+    unless record.save
+      messages = record.errors.keys.collect { |prop|
+        record.errors[prop].collect { |e|
+          v = record.send(prop)
+          "#{e} (#{prop}=#{v.inspect})."
+        }
+      }.flatten
+      fail "Could not save #{record} due to validation failures: #{messages.join(', ')}"
+    end
+    record
+  end
+
+  def create_warehouse_record_with_defaults(mdes_model, attributes={})
+    mdes_model = case mdes_model
+                 when String, Symbol
+                   mdes_model(mdes_model)
+                 else
+                   mdes_model
+                 end
+
+    all_attrs = all_missing_attributes(mdes_model).
+      merge(attributes)
+
+    save_wh(mdes_model.new(all_attrs))
+  end
+
+  def all_missing_attributes(model)
+    model.properties.
+      select { |p| p.required? }.
+      inject({}) { |h, prop| h[prop.name] = missing_value_for_property(prop); h }.
+      merge(:psu_id => '20000030')
+  end
+
+  def missing_value_for_property(dm_prop)
+    if dm_prop.options[:set]
+      '-4'
+    elsif dm_prop.class < ::DataMapper::Property::Numeric
+      0
+    else
+      'NA'
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ncs_mdes_warehouse
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-26 00:00:00.000000000 Z
+date: 2012-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ncs_mdes
@@ -1336,6 +1336,7 @@ files:
 - lib/ncs_navigator/warehouse/transformers/coded_as_missing_filter.rb
 - lib/ncs_navigator/warehouse/transformers/database.rb
 - lib/ncs_navigator/warehouse/transformers/enum_transformer.rb
+- lib/ncs_navigator/warehouse/transformers/event_start_from_contact_transformer.rb
 - lib/ncs_navigator/warehouse/transformers/filters.rb
 - lib/ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter.rb
 - lib/ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter.rb
@@ -1369,6 +1370,7 @@ files:
 - spec/ncs_navigator/warehouse/transformers/coded_as_missing_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/database_spec.rb
 - spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb
+- spec/ncs_navigator/warehouse/transformers/event_start_from_contact_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/filters_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter_spec.rb
@@ -1381,7 +1383,9 @@ files:
 - spec/ncs_navigator/warehouse/xml_emitter_spec.rb
 - spec/ncs_navigator/warehouse_spec.rb
 - spec/spec_helper.rb
+- spec/spec_warehouse_config.rb
 - spec/test_samples.csv
+- spec/warehouse_record_creation_helpers.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -1397,7 +1401,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 766009229452089340
+      hash: -1151294958631964744
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -1406,7 +1410,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 766009229452089340
+      hash: -1151294958631964744
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -1435,6 +1439,7 @@ test_files:
 - spec/ncs_navigator/warehouse/transformers/coded_as_missing_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/database_spec.rb
 - spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb
+- spec/ncs_navigator/warehouse/transformers/event_start_from_contact_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/filters_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter_spec.rb
@@ -1447,5 +1452,7 @@ test_files:
 - spec/ncs_navigator/warehouse/xml_emitter_spec.rb
 - spec/ncs_navigator/warehouse_spec.rb
 - spec/spec_helper.rb
+- spec/spec_warehouse_config.rb
 - spec/test_samples.csv
+- spec/warehouse_record_creation_helpers.rb
 has_rdoc: