ncs_mdes 0.5.0 → 0.6.0

data/documents/2.2/extract_disposition_codes.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+##
+# A script to generate disposition_codes.yml from the MDES 2.2
+# spreadsheet's Dispositions tab.
+#
+# This script requires the 'roo' gem, which is not included in
+# ncs_mdes's gemspec because it has a huge number of dependencies and
+# is not needed at runtime.
+
+require 'rubygems'
+require 'roo'
+require 'yaml'
+
+MDES_XLSX = ARGV.first or fail 'Please provide the path to the MDES spreadsheet'
+TARGET = File.expand_path('../disposition_codes.yml', __FILE__)
+SHEET_NAME = 'Dispositions'
+
+MAPPED_COLUMNS = {
+  'A' => 'final_category',
+  'B' => 'sub_category',
+  'C' => 'disposition'
+}
+
+def normalize_whitespace(s)
+  s.strip.gsub(/\s+/, " ")
+end
+
+book = Excelx.new(MDES_XLSX)
+
+# This is the array of hashes that will eventually be serialized to
+# disposition_codes.yml.
+dispositions = []
+
+current_event = nil
+current_category_code = nil
+1.upto(book.last_row(SHEET_NAME)) do |row_number|
+  a, b = %w(A B).collect { |col| book.cell(row_number, col, SHEET_NAME) }
+  if a =~ /Category\s+(\d)\s+\((.*?)\)\s+Disposition\s+Codes/
+    current_event = normalize_whitespace $2
+    current_category_code = $1.to_i
+    puts "Collecting for category #{current_event} (#{current_category_code})"
+  elsif b =~ /\S/ && a !~ /FINAL/
+    disposition_hash = MAPPED_COLUMNS.inject({}) do |h, (col, key)|
+      h[key] = normalize_whitespace(book.cell(row_number, col, SHEET_NAME)); h
+    end
+    disposition_hash['event'] = current_event
+    disposition_hash['category_code'] = current_category_code
+    disposition_hash['interim_code'], disposition_hash['final_code'] =
+      normalize_whitespace(book.cell(row_number, 'D', SHEET_NAME)).split('/')
+
+    puts disposition_hash.inspect if disposition_hash['interim_code'] =~ /^02/
+    dispositions << disposition_hash
+  end
+end
+
+File.open(TARGET, 'w') { |f| f.write dispositions.to_yaml }

data/documents/2.2/heuristic_overrides.yml

@@ -0,0 +1,452 @@
+foreign_keys:
+  birth_visit:
+    new_address_id: address
+    mail_address_id: address
+    new_address_b_id: address
+
+  birth_visit_2:
+    new_address_id: address
+    mail_address_id: address
+    new_address_b_id: address
+
+  birth_visit_baby_name:
+    bv_id: birth_visit
+
+  birth_visit_baby_name_2:
+    bv_id: birth_visit_2
+
+  birth_visit_decorate_room:
+    bv_id: birth_visit
+
+  birth_visit_decorate_room_2:
+    bv_id: birth_visit_2
+
+  birth_visit_li:
+    new_address_id: address
+    mail_address_id: address
+    new_address_b_id: address
+
+  birth_visit_renovate_room:
+    bv_id: birth_visit
+
+  birth_visit_renovate_room_2:
+    bv_id: birth_visit_2
+
+  drf_therm_verification:
+    equip_id: env_equipment
+
+  eighteen_mth_mother:
+    c_addr1_id: address
+    c_addr2_id: address
+    r_p_id: participant
+
+  eighteen_mth_mother_2:
+    c_addr1_id: address
+    c_addr2_id: address
+    r_p_id: participant
+
+  eighteen_mth_mother_cond:
+    eighteen_mth_habits_id: eighteen_mth_mother_habits
+
+  eighteen_mth_mother_saq:
+    asq18_address_id: address
+    r_p_id: participant
+
+  env_equipment_prob_log:
+    equip_id: env_equipment
+    staff_id_reviewer: staff
+
+  event:
+    participant_id: participant
+
+  household_enumeration_hidden_du:
+    hdu_address_id: address
+
+  incident:
+    inc_staff_reporter_id: staff
+    inc_staff_supervisor_id: staff
+    inc_recip_is_participant: participant
+    inc_recip_is_du: dwelling_unit
+    inc_recip_is_staff: staff
+    inc_recip_is_family: person
+    inc_recip_is_acquaintance: person
+    inc_contact_person: person
+
+  nine_mth_mother:
+    r_p_id: participant
+
+  participant_consent:
+    person_who_consented_id: person
+    person_wthdrw_consent_id: person
+
+  participant_vis_consent:
+    vis_person_who_consented_id: person
+
+  participant_rvis:
+    rvis_person: person
+
+  pb_recruitment:
+    address_pb_id: address
+
+  pb_recruitment_2:
+    address_pb_id: address
+
+  pb_recruitment_info_source:
+    pb_recruitment_id: pb_recruitment
+
+  pb_recruitment_info_source_2:
+    pb_recruitment_id: pb_recruitment_2
+
+  pb_recruitment_prov_source:
+    pb_recruitment_id: pb_recruitment
+
+  pb_recruitment_prov_source_2:
+    pb_recruitment_id: pb_recruitment_2
+
+  pb_recruitment_prov_svc:
+    pb_recruitment_id: pb_recruitment
+
+  pb_recruitment_prov_svc_2:
+    pb_recruitment_id: pb_recruitment_2
+
+  person:
+    new_address_id: address
+
+  prec_therm_cert:
+    equip_id: env_equipment
+
+  pre_preg:
+    c_addr1_id: address
+    c_addr2_id: address
+
+  preg_screen_eh:
+    mail_address_id: address
+    new_address_id: address
+
+  preg_screen_eh_2:
+    mail_address_id: address
+    new_address_id: address
+
+  preg_screen_eh_know_ncs:
+    ps_eh_id: preg_screen_eh
+
+  preg_screen_eh_know_ncs_2:
+    ps_eh_id: preg_screen_eh_2
+
+  preg_screen_eh_race:
+    ps_eh_id: preg_screen_eh
+
+  preg_screen_eh_race_2:
+    ps_eh_id: preg_screen_eh_2
+
+  preg_screen_hi:
+    mail_address_id: address
+    new_address_id: address
+
+  preg_screen_hi_2:
+    mail_address_id: address
+    new_address_id: address
+
+  preg_screen_hi_know_ncs:
+    ps_hi_id: preg_screen_hi
+
+  preg_screen_hi_know_ncs_2:
+    ps_hi_id: preg_screen_hi_2
+
+  preg_screen_hi_race:
+    ps_hi_id: preg_screen_hi
+
+  preg_screen_hi_race_2:
+    ps_hi_id: preg_screen_hi_2
+
+  preg_screen_pb:
+    mail_address_id: address
+    new_address_id: address
+
+  preg_screen_pb_2:
+    mail_address_id: address
+    new_address_id: address
+
+  preg_screen_pb_know_ncs:
+    ps_pb_id: preg_screen_pb
+
+  preg_screen_pb_know_ncs_2:
+    ps_pb_id: preg_screen_pb_2
+
+  preg_screen_pb_race:
+    ps_pb_id: preg_screen_pb
+
+  preg_screen_pb_race_2:
+    ps_pb_id: preg_screen_pb_2
+
+  preg_visit_1:
+    b_address_id: address
+    c_addr1_id: address
+    c_addr2_id: address
+
+  preg_visit_1_2:
+    b_address_id: address
+    c_addr1_id: address
+    c_addr2_id: address
+
+  preg_visit_1_commute:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_commute_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_cool:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_cool_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_diagnose_2:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_diagnose_2_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_heat2:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_heat2_2:
+    pv1_id: preg_visit_2
+
+  preg_visit_1_local_trav:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_local_trav_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_nonenglish2_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_pdecorate_room:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_pdecorate_room_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_pet_type:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_pet_type_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_prenovate2_room:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_prenovate_room:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_prenovate_room_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_room_mold:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_room_mold_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_1_saq_2:
+    f_addr_id: address
+
+  preg_visit_1_saq_3:
+    f_addr_id: address
+
+  preg_visit_1_sp_race:
+    pv1_id: preg_visit_1
+
+  preg_visit_1_sp_race_2:
+    pv1_id: preg_visit_1_2
+
+  preg_visit_2:
+    b_address_id: address
+    c_addr1_id: address
+    c_addr2_id: address
+
+  preg_visit_2_2:
+    b_address_id: address
+    c_addr1_id: address
+    c_addr2_id: address
+
+  preg_visit_2_cool:
+    pv2_id: preg_visit_2
+
+  preg_visit_2_cool_2:
+    pv2_id: preg_visit_2_2
+
+  preg_visit_2_diagnose_2:
+    pv2_id: preg_visit_2
+
+  preg_visit_2_diagnose_2_2:
+    pv2_id: preg_visit_2_2
+
+  preg_visit_2_heat2:
+    pv2_id: preg_visit_2
+
+  preg_visit_2_heat2_2:
+    pv2_id: preg_visit_2_2
+
+  preg_visit_2_pdecorate2_room:
+    pv2_id: preg_visit_2
+
+  preg_visit_2_pdecorate2_room_2:
+    pv2_id: preg_visit_2_2
+
+  preg_visit_2_prenovate_room:
+    pv2_id: preg_visit_2
+
+  preg_visit_2_prenovate_room_2:
+    pv2_id: preg_visit_2_2
+
+  preg_visit_2_room_mold:
+    pv2_id: preg_visit_2
+
+  preg_visit_2_room_mold_2:
+    pv2_id: preg_visit_2_2
+
+  preg_visit_li:
+    b_address_id: address
+
+  preg_visit_li_2:
+    b_address_id: address
+
+  preg_visit_li_cool:
+    pv_li_id: preg_visit_li
+
+  preg_visit_li_cool_2:
+    pv_li_id: preg_visit_li_2
+
+  ref_freezer_verification:
+    equip_id: env_equipment
+
+  sample_receipt_store:
+    equip_id: env_equipment
+
+  sample_shipping:
+    staff_id_track: staff
+
+  six_mth_mother:
+    c_addr1_id: address
+    c_addr2_id: address
+    r_p_id: participant
+
+  six_mth_saq_formula_type:
+    six_mth_saq_id: six_mth_saq
+
+  six_mth_saq_2:
+    r_p_id: participant
+
+  six_mth_saq_formula_type_2:
+    six_mth_saq_id: six_mth_saq_2
+
+  six_mth_saq_supp:
+    six_mth_saq_id: six_mth_saq
+
+  six_mth_saq_supp_2:
+    six_mth_saq_id: six_mth_saq_2
+
+  six_mth_saq_water:
+    six_mth_saq_id: six_mth_saq
+
+  six_mth_saq_water_2:
+    six_mth_saq_id: six_mth_saq_2
+
+  spec_blood:
+    # According to the corresponding instrument, equip_id is manually entered
+    # and not a reference to another table.
+    equip_id: false
+
+  spec_blood_2:
+    # According to the corresponding instrument, equip_id is manually entered
+    # and not a reference to another table.
+    equip_id: false
+
+  spec_cord_blood_specimen:
+    spec_cord_blood_id: spec_cord_blood
+
+  spec_cord_blood_specimen_2:
+    spec_cord_blood_id: spec_cord_blood_2
+
+  spec_receipt:
+    # Insufficient info to determine what this is supposed to link to
+    storage_container_id: false
+    equip_id: spec_equipment
+
+  spec_shipping:
+    # Insufficient info to determine what this is supposed to link to
+    shipper_id: false
+
+  spec_storage:
+    equip_id: spec_equipment
+
+  staff_exp_mngmnt_tasks:
+    staff_weekly_expense_id: staff_weekly_expense
+
+  staff_exp_data_cllctn_tasks:
+    staff_weekly_expense_id: staff_weekly_expense
+
+  three_mth_mother:
+    r_p_id: participant
+
+  trh_meter_calibration:
+    equip_id: spec_equipment
+
+  twelve_mth_mother:
+    c_addr1_id: address
+    c_addr2_id: address
+    r_p_id: participant
+
+  twelve_mth_saq_2:
+    r_p_id: participant
+
+  twelve_mth_saq_formula_brand:
+    twelve_mth_saq_id: twelve_mth_saq
+
+  twelve_mth_saq_formula_brand_2:
+    twelve_mth_saq_id: twelve_mth_saq_2
+
+  twelve_mth_saq_formula_type:
+    twelve_mth_saq_id: twelve_mth_saq
+
+  twelve_mth_saq_formula_type_2:
+    twelve_mth_saq_id: twelve_mth_saq_2
+
+  twelve_mth_saq_supplement:
+    twelve_mth_saq_id: twelve_mth_saq
+
+  twelve_mth_saq_supplement_2:
+    twelve_mth_saq_id: twelve_mth_saq_2
+
+  twelve_mth_saq_water:
+    twelve_mth_saq_id: twelve_mth_saq
+
+  twelve_mth_saq_water_2:
+    twelve_mth_saq_id: twelve_mth_saq_2
+
+  twenty_four_mth_mother:
+    c_addr1_id: address
+    c_addr2_id: address
+    r_p_id: participant
+
+  twenty_four_mth_mother_2:
+    c_addr1_id: address
+    c_addr2_id: address
+    r_p_id: participant
+
+  twenty_four_mth_mother_otc:
+    twenty_four_mth_mother_id: twenty_four_mth_mother
+
+  twenty_four_mth_mother_prescr:
+    twenty_four_mth_mother_id: twenty_four_mth_mother
+
+  twenty_four_mth_mother_suppl:
+    twenty_four_mth_mother_id: twenty_four_mth_mother
+
+  twenty_four_mth_saq:
+    asq24_address_id: address
+    r_p_id: participant
+
+  

data/lib/ncs_navigator/mdes/source_documents.rb

@@ -51,6 +51,10 @@ module NcsNavigator::Mdes
           create('1.2', '1.2/Data_Transmission_Schema_V1.2.xsd')
         when '2.0'
           create('2.0', '2.0/NCS_Transmission_Schema_2.0.01.02.xml', '2.0.01.02')
+        when '2.1'
+          create('2.1', '2.1/NCS_Transmission_Schema_2.1.00.00.xsd', '2.1.00.00')
+        when '2.2'
+          create('2.2', '2.2/NCS_Transmission_Schema_2.2.01.00.xsd', '2.2.01.00')
         else
           raise "MDES #{version} is not supported by this version of ncs_mdes"
         end

data/lib/ncs_navigator/mdes/specification.rb

@@ -91,7 +91,7 @@ module NcsNavigator::Mdes
         tables.each { |t|
           fk_overrides = heuristic_overrides['foreign_keys'][t.name] || { }
           t.variables.each { |v|
-            v.resolve_foreign_key!(tables, fk_overrides[v.name], :log => @log)
+            v.resolve_foreign_key!(tables, fk_overrides[v.name], :log => @log, :in_table => t)
           }
         }
       }

data/lib/ncs_navigator/mdes/variable.rb

@@ -175,12 +175,13 @@ module NcsNavigator::Mdes
     # @return [void]
     def resolve_foreign_key!(tables, override_name=nil, options={})
       log = options[:log] || NcsNavigator::Mdes.default_logger
+      source_table = options[:in_table] ? options[:in_table].name : '[unspecified table]'
 
       case override_name
       when String
         self.table_reference = tables.detect { |t| t.name == override_name }
         unless table_reference
-          log.warn("Foreign key #{name.inspect} explicitly mapped " <<
+          log.warn("Foreign key #{name.inspect} in #{source_table} explicitly mapped " <<
             "to unknown table #{override_name.inspect}.")
         end
       when nil
@@ -192,14 +193,15 @@ module NcsNavigator::Mdes
 
         case candidates.size
         when 0
-          log.warn("Foreign key not resolvable: " <<
+          log.warn("Foreign key in #{source_table} not resolvable: " <<
             "no tables have a primary key named #{name.inspect}.")
         when 1
           self.table_reference = candidates.first
         else
           log.warn(
-            "#{candidates.size} possible parent tables found for foreign key #{name.inspect}: " <<
-            "#{candidates.collect { |c| c.name.inspect }.join(', ')}. None used due to ambiguity.")
+            "#{candidates.size} possible parent tables found for foreign key #{name.inspect} " <<
+            "in #{source_table}: #{candidates.collect { |c| c.name.inspect }.join(', ')}. " <<
+            "None used due to ambiguity.")
         end
       end
     end

data/lib/ncs_navigator/mdes/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Mdes
-    VERSION = '0.5.0'
+    VERSION = '0.6.0'
   end
 end

data/spec/ncs_navigator/mdes/source_documents_spec.rb

@@ -72,23 +72,29 @@ module NcsNavigator::Mdes
     end
 
     describe '.get' do
-      describe '1.2' do
-        subject { SourceDocuments.get('1.2') }
-
-        it 'has the correct path for the schema' do
-          subject.schema.should =~ %r{1.2/Data_Transmission_Schema_V1.2.xsd$}
-        end
+      shared_examples 'version docs' do
+        subject { SourceDocuments.get(version) }
 
         it 'has the correct path for the overrides' do
-          subject.heuristic_overrides.should =~ %r{1.2/heuristic_overrides.yml$}
+          subject.heuristic_overrides.should =~ %r{#{version}/heuristic_overrides.yml$}
         end
 
         it 'has the correct path for the disposition codes' do
-          subject.disposition_codes.should =~ %r{1.2/disposition_codes.yml$}
+          subject.disposition_codes.should =~ %r{#{version}/disposition_codes.yml$}
         end
 
         it 'is of the specified version' do
-          subject.version.should == '1.2'
+          subject.version.should == version
+        end
+      end
+
+      describe '1.2' do
+        let(:version) { '1.2' }
+
+        include_context 'version docs'
+
+        it 'has the correct path for the schema' do
+          subject.schema.should =~ %r{1.2/Data_Transmission_Schema_V1.2.xsd$}
         end
 
         it 'has no more specific specification_version' do
@@ -97,26 +103,44 @@ module NcsNavigator::Mdes
       end
 
       describe '2.0' do
-        subject { SourceDocuments.get('2.0') }
+        let(:version) { '2.0' }
+
+        include_context 'version docs'
 
         it 'has the correct path for the schema' do
           subject.schema.should =~ %r{2.0/NCS_Transmission_Schema_2.0.01.02.xml$}
         end
 
-        it 'has the correct path for the overrides' do
-          subject.heuristic_overrides.should =~ %r{2.0/heuristic_overrides.yml$}
+        it 'has a different specification_version' do
+          subject.specification_version.should == '2.0.01.02'
         end
+      end
 
-        it 'has the correct path for the disposition codes' do
-          subject.disposition_codes.should =~ %r{2.0/disposition_codes.yml$}
+      describe '2.1' do
+        let(:version) { '2.1' }
+
+        include_context 'version docs'
+
+        it 'has the correct path for the schema' do
+          subject.schema.should =~ %r{2.1/NCS_Transmission_Schema_2.1.00.00.xsd$}
         end
 
-        it 'is of the specified version' do
-          subject.version.should == '2.0'
+        it 'has a different specification_version' do
+          subject.specification_version.should == '2.1.00.00'
+        end
+      end
+
+      describe '2.2' do
+        let(:version) { '2.2' }
+
+        include_context 'version docs'
+
+        it 'has the correct path for the schema' do
+          subject.schema.should =~ %r{2.2/NCS_Transmission_Schema_2.2.01.00.xsd$}
         end
 
         it 'has a different specification_version' do
-          subject.specification_version.should == '2.0.01.02'
+          subject.specification_version.should == '2.2.01.00'
         end
       end