ncs_mdes_warehouse 0.5.0 → 0.6.0

data/lib/ncs_navigator/warehouse/transformers/coded_as_missing_filter.rb

@@ -0,0 +1,91 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # Some source data sets include unnecessary placeholder records and
+  # values. This filter excludes the following variables and records:
+  #
+  # * Entire records whose ID is one of the common MDES error codes
+  #   (-3, -4, -6, -7).
+  #
+  # * Foreign key values whose ID is one of those same MDES error
+  #   codes.
+  #
+  # * Variable values which are not required and not coded whose value
+  #   is one of these error codes (e.g., a comments field whose value
+  #   is '-7').
+  #
+  # If the `:additional_codes` option is passed to the constructor,
+  # those codes will be used in addition to the default list (i.e.,
+  # -3, -4, -6, -7).
+  #
+  # This filter may be used with or without instantiation.
+  class CodedAsMissingFilter
+    DEFAULT_MISSING_CODES = %w(-3 -4 -6 -7)
+
+    class << self
+      def call(records)
+        singleton.call(records)
+      end
+
+      private
+
+      def singleton
+        @singleton ||= CodedAsMissingFilter.new
+      end
+    end
+
+    attr_reader :missing_codes
+
+    ##
+    # @param configuration [Configuration]
+    # @param options [Hash<Symbol, Object>]
+    # @option options [Array<String>] :additional_codes a list of
+    #   other codes (beyond the defaults) which will also be treated
+    #   as "missing" for the purposes of this filter.
+    def initialize(configuration=nil, options={})
+      @missing_codes = DEFAULT_MISSING_CODES.dup
+      if options[:additional_codes]
+        @missing_codes += options[:additional_codes]
+      end
+    end
+
+    def call(records)
+      records.collect { |r| process(r) }.compact
+    end
+
+    private
+
+    def process(record)
+      return nil if missing_codes.include?(record.key.first)
+
+      remove_missing_foreign_keys(record)
+      remove_missing_noncoded_values(record)
+
+      record
+    end
+
+    def remove_missing_foreign_keys(record)
+      record.class.relationships.each do |rel|
+        reference_key = rel.child_key.first.name
+        reference_value = record.send(reference_key)
+
+        if reference_value && missing_codes.include?(reference_value)
+          record.send("#{reference_key}=", nil)
+        end
+      end
+    end
+
+    def remove_missing_noncoded_values(record)
+      record.class.properties.each do |prop|
+        unless prop.required? || prop.options[:set]
+          prop_value = record.send(prop.name)
+          if missing_codes.include?(prop_value)
+            record.send("#{prop.name}=", nil)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/database.rb

@@ -406,7 +406,7 @@ module NcsNavigator::Warehouse::Transformers
       #   enumerable.
       # @see EnumTransformer
       def create_transformer(configuration, options={})
-        EnumTransformer.new(configuration, new(configuration, options))
+        EnumTransformer.new(configuration, new(configuration, options), options)
       end
     end
 

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -31,16 +31,31 @@ module NcsNavigator::Warehouse::Transformers
     # @return [Enumerable] the enumeration that will be transformed.
     attr_reader :enum
 
+    ##
+    # @return [Filters] the filters in use on this transformer.
+    attr_reader :filters
+
     def_delegators :@configuration, :log, :shell
 
     ##
     # @param [Configuration] configuration
     # @param [Enumerable] enum
-    def initialize(configuration, enum)
+    # @param [Hash<Symbol, Object>] options
+    # @option options [Array<#call>,#call] :filters a list of
+    #   filters to use for this transformer
+    #
+    # @see Filters
+    def initialize(configuration, enum, options={})
       @configuration = configuration
       @enum = enum
+      filter_list = options.delete(:filters)
+      @filters = Filters.new(filter_list ? [*filter_list].compact : [])
     end
 
+    ##
+    # A human-readable name for this transformer.
+    #
+    # @return [String]
     def name
       enum_name = if enum.respond_to?(:name)
                     enum.name
@@ -74,14 +89,19 @@ module NcsNavigator::Warehouse::Transformers
         when NcsNavigator::Warehouse::TransformError
           receive_transform_error(record, status)
         else
-          save_model_instance(record, status)
+          filters.call([record]).each do |filtered_record|
+            save_model_instance(filtered_record, status)
+          end
         end
       end
     end
 
     def save_model_instance(record, status)
-      apply_global_values_if_necessary(record)
-      if record.valid?
+      if !has_valid_psu?(record)
+        msg = "Invalid PSU ID #{record.psu_id.inspect}. The list of valid PSU IDs for this Study Center is #{@configuration.navigator.psus.collect(&:id).inspect}."
+        log.error "#{record_ident record}: #{msg}"
+        status.unsuccessful_record(record, msg)
+      elsif record.valid?
         log.debug("Saving valid record #{record_ident record}.")
         begin
           unless record.save
@@ -122,17 +142,16 @@ module NcsNavigator::Warehouse::Transformers
       }.flatten
     end
 
-    def apply_global_values_if_necessary(record)
-      {
-        :psu_id => @configuration.navigator.psus.first.id,
-        :recruit_type => @configuration.navigator.recruitment_type_id
-      }.each do |attr, value|
-        setter = :"#{attr}="
-        if record.respond_to?(setter) && record.respond_to?(attr)
-          unless record.send(attr)
-            record.send(setter, value)
-          end
-        end
+    ##
+    # Has valid PSU is true if:
+    #  * The record has no PSU reference, or
+    #  * The record's PSU ID is one of those configured for the
+    #    study center
+    def has_valid_psu?(record)
+      if record.respond_to?(:psu_id)
+        @configuration.navigator.psus.collect(&:id).include?(record.psu_id)
+      else
+        true
       end
     end
   end

data/lib/ncs_navigator/warehouse/transformers/filters.rb

@@ -0,0 +1,66 @@
+require 'forwardable'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # Encapsulates an ordered list of filters that can be applied to a
+  # record coming out of an enumeration in {EnumTransformer}.
+  #
+  # Each filter is an object with an `call` method. The `call`
+  # method takes an array of zero or more records as input and returns
+  # an array of zero or more records. The input and output arrays may
+  # be the same or different. A filter may add, remove, or mutate
+  # records (or all three).
+  #
+  # A filter will be called multiple times for a single transform run,
+  # but it will never see the same record twice. To put it another
+  # way, the first filter will be invoked exactly once per record
+  # yielded from an {EnumTransformer}'s enumeration. Subsequent
+  # filters are invoked on return value from the previous filter.
+  #
+  # Depending on the filter order, a particular filter may never see
+  # some of the eventually transformed records. This will happen if
+  # records are created by a filter lower in the filter chain.
+  class Filters
+    include Enumerable
+    extend Forwardable
+
+    def_delegators :@filters, :each
+
+    attr_accessor :filters
+
+    def initialize(filter_objects)
+      check_filters(filter_objects)
+
+      @filters = filter_objects
+    end
+
+    ##
+    # @param records [Array] zero or more records to filter.
+    # @return [Array] the filtered records.
+    def call(records)
+      filters.inject(ensure_array(records)) { |result, filter| ensure_array(filter.call(result)) }
+    end
+
+    private
+
+    def check_filters(candidates)
+      candidates.each_with_index do |f, i|
+        unless f.respond_to?(:call)
+          fail "Filter #{i} (#{f.class}) does not have a call method"
+        end
+      end
+    end
+
+    def ensure_array(records)
+      case records
+      when nil
+        []
+      when Enumerable
+        records.to_a
+      else
+        [records]
+      end
+    end
+  end
+end
+

data/lib/ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter.rb

@@ -0,0 +1,29 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # This filter transforms any blank foreign keys into nil. Blank
+  # foreign keys are a common issue in source XML data.
+  #
+  # This filter is stateless and so should not be instantiated when
+  # added to a warehouse's configuration.
+  class NoBlankForeignKeysFilter
+    def self.call(records)
+      records.each do |record|
+        remove_missing_foreign_keys(record)
+      end
+    end
+
+    private
+
+    def self.remove_missing_foreign_keys(record)
+      record.class.relationships.each do |rel|
+        reference_key = rel.child_key.first.name
+        reference_value = record.send(reference_key)
+        if reference_value && reference_value.strip.empty?
+          record.send("#{reference_key}=", nil)
+        end
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter.rb

@@ -0,0 +1,102 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # A filter which duplicates outreach event records that don't have
+  # an SSU ID across all of a center's SSUs.
+  #
+  # This filter is intended for use when reading a center's VDR XML
+  # into the warehouse for eventual import into a new NCS Navigator
+  # instance. Some centers have outreach events which are not
+  # associated with any SSUs in their XML. They intend this in
+  # indicate that the outreach events are associated with all the
+  # center's SSUs. This filter makes that happen.
+  #
+  # This filter is stateful and so needs to be instantiated when being
+  # added to the warehouse configuration.
+  class NoSsuOutreachAllSsusFilter
+    attr_reader :ssu_ids
+
+    ##
+    # @param configuration [Configuration] the warehouse
+    #   configuration.
+    # @param options [Hash<Symbol, Object>]
+    # @option options [Array<String>] :ssu_ids by default, full set of
+    #   SSU IDs is derived from the navigator suite configuration. It
+    #   can be overridden with this option.
+    def initialize(configuration, options={})
+      @configuration = configuration
+      @ssu_ids = options[:ssu_ids] || default_ssu_ids
+
+      @outreach_model = configuration.models_module.const_get(:Outreach)
+      @outreach_associated_models = configuration.models_module.mdes_order.select { |model|
+        model.relationships.detect { |rel| rel.parent_model == @outreach_model }
+      }
+
+      @replicated_outreach_event_ids = {}
+    end
+
+    def call(records)
+      [].tap do |result|
+        records.each do |rec|
+          if is_no_ssu_outreach?(rec)
+            result.concat replicate_outreach_across_ssus(rec)
+          elsif is_replicated_oe_associate?(rec)
+            result.concat replicate_associated_record(rec)
+          else
+            result << rec
+          end
+        end
+      end
+    end
+
+    private
+
+    def default_ssu_ids
+      @configuration.navigator.ssus.collect(&:id)
+    end
+
+    def is_no_ssu_outreach?(record)
+      record.class == @outreach_model && record.ssu_id.nil?
+    end
+
+    def is_replicated_oe_associate?(record)
+      @outreach_associated_models.include?(record.class) &&
+        @replicated_outreach_event_ids.keys.include?(record.outreach_event_id)
+    end
+
+    def replicate_outreach_across_ssus(outreach_event)
+      oe_id = outreach_event.outreach_event_id
+
+      replica_ids = ssu_ids.collect { |ssu_id| [ssu_id, create_replica_id(oe_id, ssu_id)] }
+
+      @replicated_outreach_event_ids[oe_id] = replica_ids
+
+      replica_ids.collect { |ssu_id, replica_oe_id|
+        outreach_event.class.new(
+          outreach_event.attributes.
+            merge(:ssu_id => ssu_id, :outreach_event_id => replica_oe_id)
+        )
+      }
+    end
+
+    def create_replica_id(base_id, ssu_id)
+      [base_id, ssu_id].join('-')
+    end
+
+    def replicate_associated_record(record)
+      replica_oe_ids = @replicated_outreach_event_ids[record.outreach_event_id]
+
+      associated_key_name = record.class.key.first.name
+
+      replica_oe_ids.collect { |ssu_id, replica_oe_id|
+        record.class.new(
+          record.attributes.merge(
+            :outreach_event_id => replica_oe_id,
+            associated_key_name => create_replica_id(record.key.first, ssu_id)
+          )
+        )
+      }
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/no_ssu_outreach_placeholder_filter.rb

@@ -0,0 +1,61 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # A filter which associates outreach event records that don't have
+  # an SSU ID with an automatically-created placeholder SSU.
+  #
+  # This filter is intended for use when reading a center's VDR XML
+  # into the warehouse for eventual import into a new NCS Navigator
+  # instance. Some centers have outreach events which are not
+  # logically associated with any SSUs. This scenario is permitted in
+  # MDES 2.1+, but not in MDES 2.0. This filter provides a shim to
+  # allow their data to be loaded, both into the warehouse and
+  # eventually into Staff Portal. The shim will be removed after they
+  # are upgraded to MDES 2.1.
+  #
+  # This filter is stateful and so needs to be instantiated when being
+  # added to the warehouse configuration.
+  class NoSsuOutreachPlaceholderFilter
+    PLACEHOLDER_SSU_ID = 'NO_SSU_PLACEHOLDER_PRE_MDES_2_1'
+
+    def initialize(configuration)
+      @configuration = configuration
+      @outreach_model = configuration.models_module.const_get(:Outreach)
+    end
+
+    def call(records)
+      return records unless has_any_no_ssu_outreach?(records)
+
+      unless @placeholder_ssu_created
+        records.unshift(create_placeholder_ssu)
+        @placeholder_ssu_created = true
+      end
+
+      records.each do |rec|
+        rec.ssu_id = PLACEHOLDER_SSU_ID if is_no_ssu_outreach?(rec)
+      end
+
+      records
+    end
+
+    private
+
+    def has_any_no_ssu_outreach?(records)
+      records.detect { |r| is_no_ssu_outreach?(r) }
+    end
+
+    def is_no_ssu_outreach?(record)
+      record.class == @outreach_model && record.ssu_id.nil?
+    end
+
+    def create_placeholder_ssu
+      @configuration.models_module.const_get(:Ssu).new(
+        :ssu_id => PLACEHOLDER_SSU_ID,
+        :ssu_name => 'Placeholder SSU for no-SSU outreach events until upgraded to MDES 2.0',
+        :psu_id => @configuration.navigator.psus.first.id,
+        :sc_id  => @configuration.navigator.study_center_id
+      )
+    end
+  end
+end