ncs_mdes 0.11.0 → 0.12.0

data/documents/3.2/child_or_parent_instrument_tables.yml

@@ -0,0 +1,192 @@
+child_instrument_tables:
+- birth_visit_baby_name_2
+- birth_visit_baby_name_3
+- birth_visit_baby_name_4
+- birth_visit_li_baby_name
+- birth_visit_li_baby_name_2
+- bitsea_saq
+- child_anthro
+- child_blood
+- child_bp
+- child_saliva
+- child_saliva_saq
+- child_urine
+- core_quest_child_care
+- core_quest_concern
+- core_quest_disability
+- core_quest_emergency_room
+- core_quest_health_care
+- core_quest_hh
+- core_quest_hospitalizations
+- core_quest_housing
+- core_quest_income
+- core_quest_insurance
+- core_quest_interim_med
+- core_quest_media
+- core_quest_medical
+- core_quest_neighborhood
+- core_quest_occupation
+- core_quest_pesticide
+- core_quest_pets
+- core_quest_program
+- core_quest_sleep
+- core_quest_smoke
+- core_quest_well_child_care
+- eighteen_mth_mother_detail
+- eighteen_mth_mother_detail_2
+- eighteen_mth_mother_habits
+- eighteen_mth_mother_habits_2
+- eighteen_mth_mother_habits_3
+- eighteen_mth_mother_mold
+- eighteen_mth_mother_mold_2
+- eighteen_mth_mother_mold_3
+- eighteen_mth_mother_otc
+- eighteen_mth_mother_otc_2
+- eighteen_mth_mother_otc_3
+- eighteen_mth_mother_prescr
+- eighteen_mth_mother_prescr_2
+- eighteen_mth_mother_prescr_3
+- eighteen_mth_mother_saq
+- eighteen_mth_mother_saq_2
+- eighteen_mth_mother_suppl
+- eighteen_mth_mother_suppl_2
+- eighteen_mth_mother_suppl_3
+- fourteen_mth_asq_saq
+- itsp_saq
+- m_chat_saq
+- nine_mth_mother_detail
+- nine_mth_mother_detail_2
+- participant_verif_child
+- reconsideration_ins
+- six_mth_mother_detail
+- six_mth_mother_detail_2
+- six_mth_saq
+- six_mth_saq_2
+- six_mth_saq_3
+- six_mth_saq_4
+- sixteen_mth_asq_saq
+- spec_cord_blood
+- spec_cord_blood_2
+- spec_cord_blood_3
+- thirty_month_interview_child
+- thirty_mth_asq_saq
+- three_mth_mother_child_detail
+- three_mth_mother_child_detail_2
+- three_mth_mother_child_habits
+- three_mth_mother_child_habits_2
+- three_mth_mother_child_habits_3
+- twelve_mth_mother_detail
+- twelve_mth_mother_detail_2
+- twelve_mth_mother_detail_3
+- twelve_mth_mother_mold_3
+- twenty_four_mth_mother_detail
+- twenty_four_mth_mother_detail_2
+- twenty_four_mth_mother_habits_2
+- twenty_four_mth_mother_habits_3
+- twenty_four_mth_mother_mold
+- twenty_four_mth_mother_mold_2
+- twenty_four_mth_mother_otc_2
+- twenty_four_mth_mother_otc_3
+- twenty_four_mth_mother_prescr_2
+- twenty_four_mth_mother_prescr_3
+- twenty_four_mth_mother_suppl_2
+- twenty_four_mth_mother_suppl_3
+- twenty_four_mth_saq
+- twenty_four_mth_saq_2
+- twenty_mth_asq_saq
+- twenty_seven_mth_asq_saq
+- twenty_two_mth_asq_saq
+parent_instrument_tables:
+- birth_visit
+- birth_visit_2
+- birth_visit_3
+- birth_visit_4
+- birth_visit_diagnose_2_3
+- birth_visit_diagnose_2_4
+- birth_visit_household_3
+- birth_visit_household_4
+- birth_visit_li
+- birth_visit_li_2
+- breast_milk_saq
+- bsi_saq
+- eighteen_mth_mother
+- eighteen_mth_mother_2
+- eighteen_mth_mother_3
+- father_pv1
+- father_pv1_2
+- household_enumeration
+- household_enumeration_age_elig
+- household_enumeration_pregnant
+- household_inventory
+- household_inventory_age
+- household_inventory_age_elig
+- internet_usage
+- low_high_script
+- multi_mode
+- nine_mth_mother
+- nine_mth_mother_2
+- non_interview_respondent
+- participant_verif
+- pbs_elig_screener
+- pbs_participant_verif
+- ppg_cati
+- ppg_saq
+- pre_preg
+- pre_preg_saq
+- preg_screen_eh
+- preg_screen_eh_2
+- preg_screen_hi
+- preg_screen_hi_2
+- preg_screen_pb
+- preg_screen_pb_2
+- preg_visit_1
+- preg_visit_1_2
+- preg_visit_1_3
+- preg_visit_1_saq
+- preg_visit_1_saq_2
+- preg_visit_1_saq_3
+- preg_visit_1_saq_4
+- preg_visit_2
+- preg_visit_2_2
+- preg_visit_2_3
+- preg_visit_2_saq
+- preg_visit_2_saq_2
+- preg_visit_2_saq_3
+- preg_visit_2_saq_4
+- preg_visit_li
+- preg_visit_li_2
+- sample_dist
+- six_mth_mother
+- six_mth_mother_2
+- spec_blood
+- spec_blood_2
+- spec_urine
+- tap_water_twf
+- tap_water_twf_2
+- tap_water_twf_saq
+- tap_water_twq
+- tap_water_twq_2
+- tap_water_twq_saq
+- thirty_month_interview
+- three_mth_mother
+- three_mth_mother_2
+- three_mth_mother_3
+- tracing_int
+- twelve_mth_mother
+- twelve_mth_mother_2
+- twelve_mth_mother_3
+- twelve_mth_saq
+- twelve_mth_saq_2
+- twelve_mth_saq_3
+- twenty_four_mth_mother
+- twenty_four_mth_mother_2
+- twenty_four_mth_mother_3
+- twenty_four_mth_mother_habits
+- twenty_four_mth_mother_otc
+- twenty_four_mth_mother_prescr
+- twenty_four_mth_mother_suppl
+- vacuum_bag
+- vacuum_bag_2
+- vacuum_bag_saq
+- validation_ins
+- validation_ins_2

data/documents/scan_p_ids_for_children.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+##
+# A script which scans an MDES spreadsheet for indications that an instrument
+# table's p_id is for a child (vs. for a parent.)
+#
+# This script requires the 'roo' gem, which is not included in
+# ncs_mdes's gemspec because it has a huge number of dependencies and
+# is not needed at runtime.
+
+require 'roo'
+require 'yaml'
+
+MDES_XLSX = ARGV.first or fail 'Please provide the path to the MDES spreadsheet'
+SHEET_NAME = 'Data Elements'
+
+COLUMNS = {
+  'A' => :table_type,
+  'B' => :table_label,
+  'C' => :table_name,
+  'D' => :variable_name,
+  'I' => :variable_def
+}
+
+book = Excelx.new(MDES_XLSX)
+
+3.upto(book.last_row(SHEET_NAME)) do |row_number|
+  row = COLUMNS.keys.each_with_object({}) { |col, i| i[COLUMNS[col]] = book.cell(row_number, col, SHEET_NAME) }
+  next unless row[:table_type] =~ /instrument/i
+  next unless row[:variable_name] =~ /\Ap_id\Z/i
+
+  puts "#{row[:table_name].downcase}.#{row[:variable_name].downcase}"
+  if row[:variable_def] =~ /child/i
+    puts "- mentions child in variable def"
+  end
+  if row[:table_label] =~ /child/i
+    puts "- mentions child in table label"
+  end
+end

data/lib/ncs_navigator/mdes/code_list.rb

@@ -0,0 +1,94 @@
+require 'ncs_navigator/mdes'
+
+module NcsNavigator::Mdes
+  ##
+  # A specialization of `Array` for code lists.
+  #
+  # @see VariableType#code_list
+  # @see CodeListEntry
+  class CodeList < Array
+    ##
+    # @return [String,nil] the description of the code list if any.
+    attr_accessor :description
+  end
+
+  ##
+  # A single entry in a code list.
+  #
+  # @see VariableType#code_list
+  # @see CodeList
+  class CodeListEntry
+    ##
+    # @return [String] the local code value for the entry.
+    attr_reader :value
+
+    ##
+    # @return [String] the human-readable label for the entry.
+    attr_accessor :label
+
+    ##
+    # @return [String] the MDES's globally-unique identifier for
+    #   this coded value.
+    attr_accessor :global_value
+
+    ##
+    # @return [String] the name of MDES's master code list from
+    #   which this value is derived.
+    attr_accessor :master_cl
+
+    class << self
+      ##
+      # Creates a new instance from a `xs:enumeration` simple type
+      # restriction subelement.
+      #
+      # @param [Nokogiri::XML::Element] enum the `xs:enumeration`
+      #   element.
+      # @param [Hash] options
+      # @option options [#warn] :log the logger to which to direct warnings
+      #
+      # @return [CodeListEntry]
+      def from_xsd_enumeration(enum, options={})
+        log = options[:log] || NcsNavigator::Mdes.default_logger
+
+        log.warn("Missing value for code list entry on line #{enum.line}") unless enum['value']
+
+        new(enum['value'] && enum['value'].strip).tap do |cle|
+          cle.label = enum['ncsdoc:label']
+          cle.global_value = enum['ncsdoc:global_value']
+          cle.master_cl = enum['ncsdoc:master_cl']
+        end
+      end
+    end
+
+    def initialize(value)
+      @value = value
+    end
+
+    alias :to_s :value
+
+    def diff_criteria(diff_options)
+      if diff_options[:strict]
+        {
+          :value        => Differences::ValueCriterion.new,
+          :label        => Differences::ValueCriterion.new,
+          :global_value => Differences::ValueCriterion.new,
+          :master_cl    => Differences::ValueCriterion.new
+        }
+      else
+        {
+          :value => Differences::ValueCriterion.new,
+          :label => Differences::ValueCriterion.new(:value_extractor => :word_chars_downcase)
+        }
+      end
+    end
+    protected :diff_criteria
+
+    ##
+    # Computes the differences between this code list entry and the other.
+    #
+    # @return [Differences::Entry,nil]
+    def diff(other, options={})
+      Differences::Entry.compute(self, other, diff_criteria(options), options)
+    end
+  end
+end

data/lib/ncs_navigator/mdes/differences/collection.rb

@@ -0,0 +1,47 @@
+require 'ncs_navigator/mdes'
+require 'forwardable'
+
+module NcsNavigator::Mdes::Differences
+  ##
+  # Captures the differences between two collections.
+  class Collection
+    extend Forwardable
+
+    def_delegators :entry_differences, :[]
+
+    def initialize(left_only, right_only, entry_differences)
+      @left_only = left_only
+      @right_only = right_only
+      @entry_differences = entry_differences
+    end
+
+    ##
+    # A list of those entries which are in the lefthand version of the
+    # collection only. Values are the characteristic (alignment) value for each
+    # entry.
+    #
+    # @return [Array<Object>]
+    def left_only
+      @left_only ||= []
+    end
+
+    ##
+    # A list of those entries which are in the righthand version of the
+    # collection only. Values are the characteristic (alignment) value for each
+    # entry.
+    #
+    # @return [Array<Object>]
+    def right_only
+      @right_only ||= []
+    end
+
+    ##
+    # Detailed differences for entries which are present in some form in each
+    # collection. Keys are the characteristic (alignment) value for the entry.
+    #
+    # @return [Hash<Object, Entry>]
+    def entry_differences
+      @entry_differences ||= {}
+    end
+  end
+end

data/lib/ncs_navigator/mdes/differences/collection_criterion.rb

@@ -0,0 +1,59 @@
+require 'ncs_navigator/mdes'
+
+module NcsNavigator::Mdes::Differences
+  ##
+  # @private implementation detail
+  class CollectionCriterion
+    ##
+    # @return [Symbol] the attribute in the object to which this criterion applies
+    attr_reader :attribute
+
+    def initialize(alignment_attribute, options={})
+      @alignment_attribute = alignment_attribute
+      @attribute = options.delete(:collection)
+      @alignment_value_extractor =
+        select_value_extractor(options.delete(:value_extractor))
+    end
+
+    def apply(c1, c2, diff_options)
+      c1_map = map_for_alignment(c1)
+      c2_map = map_for_alignment(c2)
+
+      left_only = c1_map.keys - c2_map.keys
+      right_only = c2_map.keys - c1_map.keys
+
+      both = c1_map.keys & c2_map.keys
+      entry_differences = both.each_with_object({}) do |key, differences|
+        diff = c1_map[key].diff(c2_map[key], diff_options)
+        differences[key] = diff if diff
+      end
+
+      if left_only.empty? && right_only.empty? && entry_differences.empty?
+        nil
+      else
+        Collection.new(left_only, right_only, entry_differences)
+      end
+    end
+
+    def map_for_alignment(c)
+      return {} unless c
+      c.each_with_object({}) do |element, map|
+        value = @alignment_value_extractor.call(element.send(@alignment_attribute))
+        map[value] = element
+      end
+    end
+    private :map_for_alignment
+
+    def select_value_extractor(param)
+      case param
+      when nil
+        ValueCriterion::VALUE_EXTRACTORS[:identity]
+      when Symbol
+        ValueCriterion::VALUE_EXTRACTORS[param] or fail "Unknown extractor #{param.inspect}"
+      else
+        param
+      end
+    end
+    private :select_value_extractor
+  end
+end

data/lib/ncs_navigator/mdes/differences/entry.rb

@@ -0,0 +1,48 @@
+require 'ncs_navigator/mdes'
+require 'forwardable'
+
+module NcsNavigator::Mdes::Differences
+  ##
+  # Captures the differences between two instances of one of the elements that
+  # makes up a specification. I.e., {TransmissionTable}, {Variable},
+  # {VariableType}, or {CodeListEntry}.
+  class Entry
+    ##
+    # @param [Object] o1 the left object
+    # @param [Object] o2 the right object
+    # @param [Hash<Symbol, #apply>] attribute_criteria a list of objects which produce difference objects
+    # @param [Hash] diff_options options to pass to nested calls to #diff
+    # @return [Entry, nil] the differences between o1 and o2 according to the
+    #   criteria, or nil if there are no differences.
+    def self.compute(o1, o2, attribute_criteria, diff_options)
+      differences = attribute_criteria.each_with_object({}) do |(diff_attribute, criterion), diffs|
+        o_attribute = (criterion.respond_to?(:attribute) && criterion.attribute) || diff_attribute
+        d = criterion.apply(o1.send(o_attribute), o2.send(o_attribute), diff_options)
+        diffs[diff_attribute] = d if d
+      end
+
+      if differences.empty?
+        nil
+      else
+        Entry.new.tap { |e| e.attribute_differences = differences }
+      end
+    end
+
+    extend Forwardable
+
+    def_delegators :attribute_differences, :[]
+
+    ##
+    # Return the differences for each attribute. Each key is the name of the
+    # attribute and each value is an object describing the difference. It might
+    # be a {Value} diff, a {Collection} diff, or another {Entry} diff depending
+    # on the kind of attribute.
+    #
+    # @return [Hash<Symbol, Object>]
+    def attribute_differences
+      @attribute_differences ||= {}
+    end
+
+    attr_writer :attribute_differences
+  end
+end

data/lib/ncs_navigator/mdes/differences/value.rb

@@ -0,0 +1,7 @@
+require 'ncs_navigator/mdes'
+
+module NcsNavigator::Mdes::Differences
+  ##
+  # Captures the differences between a simple scalar value.
+  Value = Struct.new(:left, :right)
+end

data/lib/ncs_navigator/mdes/differences/value_criterion.rb

@@ -0,0 +1,58 @@
+require 'ncs_navigator/mdes'
+
+module NcsNavigator::Mdes::Differences
+  ##
+  # @private implementation detail
+  class ValueCriterion
+    COMPARATORS = {
+      :equality => lambda { |a, b| a == b },
+      :predicate => lambda { |a, b| !(a ^ b) }
+    }
+
+    VALUE_EXTRACTORS = {
+      :identity => lambda { |o| o },
+      :word_chars_downcase =>
+        lambda { |o| o ? o.downcase.gsub(/[^ \w]+/, ' ').gsub(/\s+/, ' ').strip : o }
+    }
+
+    attr_reader :comparator, :value_extractor
+
+    def initialize(options={})
+      @comparator = select_comparator(options.delete(:comparator))
+      @value_extractor = select_value_extractor(options.delete(:value_extractor))
+    end
+
+    def apply(v1, v2, diff_options)
+      cv1 = value_extractor.call(v1)
+      cv2 = value_extractor.call(v2)
+
+      unless comparator.call(cv1, cv2)
+        Value.new(cv1, cv2)
+      end
+    end
+
+    def select_comparator(param)
+      case param
+      when nil
+        COMPARATORS[:equality]
+      when Symbol
+        COMPARATORS[param] or fail "Unknown comparator #{param.inspect}"
+      else
+        param
+      end
+    end
+    private :select_comparator
+
+    def select_value_extractor(param)
+      case param
+      when nil
+        VALUE_EXTRACTORS[:identity]
+      when Symbol
+        VALUE_EXTRACTORS[param] or fail "Unknown extractor #{param.inspect}"
+      else
+        param
+      end
+    end
+    private :select_value_extractor
+  end
+end

data/lib/ncs_navigator/mdes/differences.rb

@@ -0,0 +1,12 @@
+require 'ncs_navigator/mdes'
+
+module NcsNavigator::Mdes
+  module Differences
+    autoload :Entry,      'ncs_navigator/mdes/differences/entry'
+    autoload :Collection, 'ncs_navigator/mdes/differences/collection'
+    autoload :Value,      'ncs_navigator/mdes/differences/value'
+
+    autoload :CollectionCriterion, 'ncs_navigator/mdes/differences/collection_criterion'
+    autoload :ValueCriterion,      'ncs_navigator/mdes/differences/value_criterion'
+  end
+end

data/lib/ncs_navigator/mdes/source_documents.rb

@@ -72,6 +72,7 @@ module NcsNavigator::Mdes
           sd.schema = schema
           sd.heuristic_overrides = "#{version}/heuristic_overrides.yml"
           sd.disposition_codes = "#{version}/disposition_codes.yml"
+          sd.child_or_parent_instrument_tables = "#{version}/child_or_parent_instrument_tables.yml"
           sd.specification_version = specification_version
         end
       end
@@ -171,6 +172,32 @@ module NcsNavigator::Mdes
       @disposition_codes = path
     end
 
+    ##
+    # The absolute path to a YAML-formatted document defining a hash with two
+    # keys: `child_instrument_tables` and `parent_instrument_tables`. The value
+    # for each should be a list of MDES table names (lower case) which are in
+    # that category.
+    #
+    # This is path is optional; if one is not provided
+    # {TransmissionTable#child_instrument_table?} and
+    # {TransmissionTable#parent_instrument_table?} will be nil for all tables.
+    #
+    # @return [String]
+    def child_or_parent_instrument_tables
+      absolutize(@child_or_parent_instrument_tables)
+    end
+
+    ##
+    # Set the path to the child-or-parent instrument tables document.
+    # If the path is relative (i.e., it does not begin with `/`), it
+    # will be interpreted relative to {#base}.
+    #
+    # @param [String] path
+    # @return [String] the provided path
+    def child_or_parent_instrument_tables=(path)
+      @child_or_parent_instrument_tables = path
+    end
+
     private
 
     def absolutize(path)

data/lib/ncs_navigator/mdes/specification.rb

@@ -94,10 +94,44 @@ module NcsNavigator::Mdes
             v.resolve_foreign_key!(tables, fk_overrides[v.name], :log => @log, :in_table => t)
           }
         }
+        tables.each { |t| t.child_instrument_table = lookup_child_or_parent_table_status(t.name) }
       }
     end
     private :read_transmission_tables
 
+    def lookup_child_or_parent_table_status(name)
+      if child_or_parent_instrument_tables['child_instrument_tables'].include?(name)
+        true
+      elsif child_or_parent_instrument_tables['parent_instrument_tables'].include?(name)
+        false
+      else
+        nil
+      end
+    end
+    private :lookup_child_or_parent_table_status
+
+    def child_or_parent_instrument_tables
+      @child_or_parent_instrument_tables ||=
+        if source_documents.child_or_parent_instrument_tables
+          YAML.load(File.read(source_documents.child_or_parent_instrument_tables)).tap do |result|
+            check_child_or_parent_lists(result)
+          end
+        else
+          Hash.new([])
+        end
+    end
+    private :child_or_parent_instrument_tables
+
+    def check_child_or_parent_lists(lists)
+      parent_list = lists['parent_instrument_tables']
+      child_list = lists['child_instrument_tables']
+      overlap = parent_list & child_list
+      unless overlap.empty?
+        @log.warn("These tables appear in both the child instrument and parent instrument lists: #{overlap.inspect}")
+      end
+    end
+    private :check_child_or_parent_lists
+
     ##
     # A shortcut for accessing particular {#transmission_tables}.
     #
@@ -163,5 +197,16 @@ module NcsNavigator::Mdes
     def inspect
       "#<#{self.class} version=#{version.inspect}>"
     end
+
+    # @private
+    DIFF_CRITERIA = {
+      :specification_version => Differences::ValueCriterion.new,
+      :transmission_tables   => Differences::CollectionCriterion.new(:name),
+      :types                 => Differences::CollectionCriterion.new(:name)
+    }
+
+    def diff(other, options={})
+      Differences::Entry.compute(self, other, DIFF_CRITERIA, options)
+    end
   end
 end