ncs_mdes_warehouse 0.10.1 → 0.11.0

data/CHANGELOG.md

@@ -1,11 +1,48 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.11.0
+------
+
+- Add `--and-pii` option to `emit-xml` to allow for simultaneously producing
+  with- and without-PII XML from the same database reads. (#2285)
+
+- Add `--directory` option to `emit-xml` to allow for writing files with the
+  default names somewhere other than the current working directory. (#3073)
+
+- Change `EnumTransformer` foreign key error handling. Foreign keys are
+  now completely resolved in memory. Previously, foreign key errors were
+  reported out of `EnumTransformer`, but the records were saved anyway. This
+  was mostly in order to allow the database to handle circular reference
+  resolution, but also because that implementation was simpler. The new
+  implementation is expected to handle resolving all possible foreign key issues
+  and so does not save records it finds to have bad FKs (or records that refer
+  to those records, etc.) (#3188)
+
+- In order to handle the previous, the interface for the object expected to be
+  in the configuration property `foreign_key_index` has changed. In the very
+  unlikely event that you were using a custom implementation, see the
+  `Configuration#foreign_key_index` docs for the new protocol.
+
+- Handle `xsi:nil` in `VdrXml::Reader`. (#3217)
+
+- Limit caught exceptions in `EnumTransformer` to `StandardError` & subclasses.
+  (#3243)
+
+- Use `attribute_name` and `attribute_value` when creating TransformErrors for
+  foreign key violations. (#3189)
+
+- Report invalid properties or unresolvable FKs on records that have invalid
+  PSU IDs. (#3264)
+
+- Clean up shell output for `Database`-based enumerators. (Includes #3072.)
+
 0.10.1
 ------
 
 - Eliminate method name collisions when a generated model would have had
   both a property and a belongs_to with the same name. (#3184)
+
 - Update MDES 3.1 models to 3.1.01.00. (#2750)
 
 0.10.0

data/lib/ncs_navigator/warehouse/cli.rb

@@ -54,10 +54,14 @@ DESC
     method_option 'block-size', :type => :numeric, :aliases => %w(-b),
       :desc => 'The maximum number of records to have in memory at once.',
       :default => 5000
+    method_option 'and-pii', :type => :boolean, :default => false,
+      :desc => 'Emit one XML file without PII and one with.'
     method_option 'include-pii', :type => :boolean, :default => false,
-      :desc => 'Include PII values in the emitted XML.'
+      :desc => 'Include PII values in the emitted XML. (Usually you should prefer --and-pii.)'
     method_option 'zip', :type => :boolean, :default => true,
       :desc => 'Create a zip file alongside the XML. (Use --no-zip to disable.)'
+    method_option 'directory', :type => :string, :default => nil,
+      :desc => 'The target directory for automatically-named files. (Default is CWD.)'
     method_option 'tables', :type => :string,
       :desc => 'Emit XML for a subset of tables.', :banner => 'TABLE,TABLE,TABLE'
     def emit_xml(filename=nil)

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -97,8 +97,8 @@ module NcsNavigator::Warehouse
     # value is correct for virtually any case.
     #
     # @return [void]
-    # @param [#record_and_verify,#report_errors] index the replacement
-    #   foreign key index implementation.
+    # @param [#start_transform,#verify_or_defer,#record,#end_transform] index
+    #   the replacement foreign key index implementation.
     def foreign_key_index=(index)
       @foreign_key_index = index
     end

data/lib/ncs_navigator/warehouse/transformers/database.rb

@@ -121,7 +121,7 @@ module NcsNavigator::Warehouse::Transformers
 
       producers.each do |rp|
         shell.clear_line_then_say(
-          "Producing records from %-#{producer_name_length}s (%-22s)" % [rp.name, 'loading'])
+          "Producing records from %-#{producer_name_length}s (%-24s)" % [rp.name, 'loading'])
         log.debug("Executing query for producer #{rp.name}:\n#{rp.query}")
         repository.adapter.select(rp.query).each do |row|
           meta = { :configuration => @configuration }
@@ -132,13 +132,14 @@ module NcsNavigator::Warehouse::Transformers
           [*rp.row_processor.call(*args)].compact.each do |result|
             yield result
             result_count += 1
-            shell.back_up_and_say(24, "(%-6d in / %-6d out)" % [row_count, result_count])
+            shell.back_up_and_say(26, "(%-7d in / %-7d out)" % [row_count, result_count])
           end
-          shell.back_up_and_say(24, "(%-6d in / %-6d out)" % [row_count, result_count])
+          shell.back_up_and_say(26, "(%-7d in / %-7d out)" % [row_count, result_count])
         end
+        shell.back_up_and_say(26, "(%-7d in / %-7d out)" % [row_count, result_count])
         log.debug("Producer #{rp.name} complete")
       end
-      shell.say_line "\nComplete"
+      shell.clear_line_then_say("#{self.class} complete (%-7d in / %-7d out)\n" % [row_count, result_count])
 
       log.info(
         "Production from #{self.class} complete. " +

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -62,6 +62,12 @@ module NcsNavigator::Warehouse::Transformers
       @filters = Filters.new(filter_list ? [*filter_list].compact : [])
       @duplicates = options.delete(:duplicates) || :error
       @duplicates_strategy = select_duplicates_strategy
+
+      @record_checkers = {
+        :validation => ValidateRecordChecker.new(log),
+        :foreign_key => ForeignKeyChecker.new(log, foreign_key_index),
+        :psus => PsuIdChecker.new(log, @configuration.navigator.psus)
+      }
     end
 
     ##
@@ -86,7 +92,7 @@ module NcsNavigator::Warehouse::Transformers
     def transform(status)
       begin
         do_transform(status)
-      rescue Exception => e
+      rescue => e
         err = NcsNavigator::Warehouse::TransformError.for_exception(e, 'Enumeration failed.')
         log.error err.message
         status.transform_errors << err
@@ -96,55 +102,137 @@ module NcsNavigator::Warehouse::Transformers
     private
 
     def do_transform(status)
+      foreign_key_index.start_transform(status)
       enum.each do |record|
         case record
         when NcsNavigator::Warehouse::TransformError
           receive_transform_error(record, status)
         else
           filters.call([record]).each do |filtered_record|
-            saved_record = save_model_instance(filtered_record, status)
-            foreign_key_index.record_and_verify(saved_record) if saved_record
+            save_model_instance(filtered_record, status, [:psus, :validation, :foreign_key])
           end
         end
       end
-      foreign_key_index.report_errors(status)
+      late_resolved_records = foreign_key_index.end_transform
+      late_resolved_records.each do |record|
+        save_model_instance(record, status, []) unless has_reported_errors?(record, status)
+      end
     end
 
-    def save_model_instance(incoming_record, status)
+    ##
+    # @return [void]
+    def save_model_instance(incoming_record, status, record_check_kinds)
+      status.record_count += 1
+
       record = process_duplicate_if_appropriate(incoming_record)
       unless record
         log.info("Ignoring duplicate record #{record_ident incoming_record}.")
-        status.record_count += 1
         return
       end
 
-      saved_record =
-        if !has_valid_psu?(record)
-          msg = "Invalid PSU ID. The list of valid PSU IDs for this Study Center is #{@configuration.navigator.psus.collect(&:id).inspect}."
+      record_checks = record_check_kinds.map { |kind| @record_checkers[kind] }
+
+      saveable = verify_record_or_report_errors(record, status, record_checks)
+
+      if saveable
+        log.debug("Saving verified record #{record_ident record}.")
+        begin
+          if record.save
+            record
+            foreign_key_index.record(record)
+          else
+            msg = "Could not save valid record #{record.inspect}."
+            log.error msg
+            status.unsuccessful_record(record, msg)
+          end
+        rescue => e
+          msg = "Error on save. #{e.class}: #{e}."
+          log.error msg
+          status.unsuccessful_record(record, msg)
+        end
+      end
+    end
+
+    def receive_transform_error(error, status)
+      error.id = nil
+      status.transform_errors << error
+    end
+
+    def has_reported_errors?(record, status)
+      status.transform_errors.any? { |error|
+        error.model_class.to_s == record.class.to_s &&
+          error.record_id.to_s == record.key.first.to_s
+      }
+    end
+
+    module RecordIdent
+      def record_ident(rec)
+        # No composite keys in the MDES
+        '%s %s=%s' % [
+          rec.class.name.demodulize, rec.class.key.first.name, rec.key.try(:first).inspect]
+      end
+    end
+    include RecordIdent
+
+    ###### CHECKING FOR ERRORS IN A RECORD
+
+    def verify_record_or_report_errors(record, status, record_checks)
+      record_checks.collect { |check| check.verify_or_report_errors(record, status) }.
+        reject { |r| r }.empty? # does the result contain anything that isn't truthy?
+    end
+
+    class PsuIdChecker
+      include RecordIdent
+
+      attr_reader :log
+
+      def initialize(log, psus)
+        @log = log
+        @psus = psus
+      end
+
+      ##
+      # Has valid PSU is true if:
+      #  * The record has no PSU reference, or
+      #  * The record's PSU ID is one of those configured for the
+      #    study center
+      def has_valid_psu?(record)
+        if record.respond_to?(:psu_id)
+          @psus.collect(&:id).include?(record.psu_id)
+        else
+          true
+        end
+      end
+
+      def verify_or_report_errors(record, status)
+        if has_valid_psu?(record)
+          true
+        else
+          msg = "Invalid PSU ID. The list of valid PSU IDs for this Study Center is #{@psus.collect(&:id).inspect}."
           log.error "#{record_ident record}: #{msg}"
           status.unsuccessful_record(record, msg,
             :attribute_name => 'psu_id',
             :attribute_value => record.psu_id.inspect)
-          nil
-        elsif record.valid?
-          log.debug("Saving valid record #{record_ident record}.")
-          begin
-            if record.save
-              record
-            else
-              msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
-              log.error msg
-              status.unsuccessful_record(record, msg)
-              nil
-            end
-          rescue => e
-            msg = "Error on save. #{e.class}: #{e}."
-            log.error msg
-            status.unsuccessful_record(record, msg)
-            nil
-          end
+          false
+        end
+      end
+    end
+
+    class ValidateRecordChecker
+      include RecordIdent
+
+      attr_reader :log
+
+      def initialize(log)
+        @log = log
+      end
+
+      def verify_or_report_errors(record, status)
+        if record.valid?
+          log.debug "#{record_ident record} is valid."
+          true
         else
-          log.error "Invalid record. #{record_messages(record).join(' ')}"
+          log.error "#{record_ident record} is not valid. #{record_messages(record).join(' ')}"
           record.errors.keys.each do |prop|
             record.errors[prop].each do |e|
               status.unsuccessful_record(
@@ -154,45 +242,44 @@ module NcsNavigator::Warehouse::Transformers
               )
             end
           end
-          nil
+          false
         end
-      status.record_count += 1
-      saved_record
-    end
+      end
 
-    def receive_transform_error(error, status)
-      error.id = nil
-      status.transform_errors << error
+      def record_messages(record)
+        record.errors.keys.collect { |prop|
+          record.errors[prop].collect { |e|
+            v = record.send(prop)
+            "#{e} (#{prop}=#{v.inspect})."
+          }
+        }.flatten
+      end
     end
 
-    def record_ident(rec)
-      # No composite keys in the MDES
-      '%s %s=%s' % [
-        rec.class.name.demodulize, rec.class.key.first.name, rec.key.try(:first).inspect]
-    end
+    class ForeignKeyChecker
+      include RecordIdent
 
-    def record_messages(record)
-      record.errors.keys.collect { |prop|
-        record.errors[prop].collect { |e|
-          v = record.send(prop)
-          "#{e} (#{prop}=#{v.inspect})."
-        }
-      }.flatten
-    end
+      attr_reader :log, :fk_index
 
-    ##
-    # Has valid PSU is true if:
-    #  * The record has no PSU reference, or
-    #  * The record's PSU ID is one of those configured for the
-    #    study center
-    def has_valid_psu?(record)
-      if record.respond_to?(:psu_id)
-        @configuration.navigator.psus.collect(&:id).include?(record.psu_id)
-      else
-        true
+      def initialize(log, fk_index)
+        @log = log
+        @fk_index = fk_index
+      end
+
+      def verify_or_report_errors(record, status)
+        log.debug "Verifying FKs for #{record_ident record}"
+        fk_index.verify_or_defer(record).tap do |result|
+          if result
+            log.debug "- All FKs currently resolved."
+          else
+            log.debug "- Deferring because one or more FKs are not resolved."
+          end
+        end
       end
     end
 
+    ###### HANDLING DUPLICATES
+
     def process_duplicate_if_appropriate(record)
       if @duplicates_strategy.duplicate?(record)
         @duplicates_strategy.to_save(record)

data/lib/ncs_navigator/warehouse/transformers/foreign_key_index.rb

@@ -41,27 +41,56 @@ module NcsNavigator::Warehouse::Transformers
         DatabaseKeyProvider.new
       end
       @seen_keys = {}
+      @current_transform_tracker = nil
     end
 
     ##
-    # Records the key for this record in the index and verifies its
-    # foreign keys. If any are not satisfied, they will be stored for
-    # later evaluation.
+    # Indicates the beginning of a new transform. This method must be called
+    # before the first time {#verify_or_defer} is called for a particular transform.
+    #
+    # Error reporting and deferred foreign key resolution are scoped to a
+    # transform.
+    def start_transform(transform_status)
+      if @current_transform_tracker
+        fail "#start_transform called before previous transform's #end_transform called. This will lose deferred records."
+      else
+        @current_transform_tracker = TransformTracker.new(transform_status)
+      end
+    end
+
+    ##
+    # Indicates whether a record's foreign keys can immediately be satisfied. If
+    # not, it stores the record for processing in and possible return from
+    # {#end_transform}.
+    #
+    # @param [DataMapper::Resource] record the record whose foreign references
+    #   we want to verify.
+    # @return [Boolean]
+    def verify_or_defer(record)
+      deferrer = DeferredRecord.create_if_appropriate(record, self)
+      if deferrer
+        @current_transform_tracker.defer(deferrer)
+        false
+      else
+        true
+      end
+    end
+
+    ##
+    # Records the key for this record in the index. By calling this method,
+    # the caller affirms that the record should be considered available for
+    # resolution of future foreign keys.
     #
     # @param [DataMapper::Resource] record the record whose key we
-    #   want to record and whose foreign references we want to verify.
+    #   want to record.
     # @return [void]
-    def record_and_verify(record)
+    def record(record)
       seen_keys(record.class) << record.key.first # no CPKs in MDES
-
-      record.class.relationships.each do |belongs_to|
-        verify_relationship(record, belongs_to)
-      end
     end
 
     ##
     # Reviews any references that initially failed against the final
-    # set of keys. If any are still unresolveable, it records errors
+    # set of keys. If any are still unresolvable, it records errors
     # against the provided transform status.
     #
     # Each failed reference will be reported by this method only
@@ -69,15 +98,14 @@ module NcsNavigator::Warehouse::Transformers
     # this class across multiple transformers, reporting only the
     # newly encountered errors for each transform in turn.
     #
-    # @return [void]
-    # @param [TransformStatus] transform_status
-    def report_errors(transform_status)
-      interim_unsatisfied.each do |relationship_instance|
-        if !seen?(relationship_instance.foreign_model, relationship_instance.reference_value)
-          transform_status.transform_errors << relationship_instance.create_error
-        end
+    # @return [Array<DataMapper::Resource>] any records which were previously
+    #   deferred but which now are fully resolved and are candidates to be
+    #   persisted.
+    def end_transform
+      fail "No current transform" unless @current_transform_tracker
+      @current_transform_tracker.end_it(self).tap do |x|
+        @current_transform_tracker =  nil
       end
-      interim_unsatisfied.clear
     end
 
     ##
@@ -89,18 +117,6 @@ module NcsNavigator::Warehouse::Transformers
 
     private
 
-    def verify_relationship(record, belongs_to)
-      reference_name  = belongs_to.child_key.first.name
-      reference_value = record.send(reference_name)
-      foreign_model   = belongs_to.parent_model
-
-      if reference_value && !seen?(foreign_model, reference_value)
-        interim_unsatisfied << RelationshipInstance.new(
-          record.key.first, record.class.to_s, foreign_model, reference_name, reference_value
-        )
-      end
-    end
-
     def seen_keys(model_class)
       @seen_keys[model_class.to_s] ||= begin
         existing_keys = existing_key_provider.existing_keys(model_class) || []
@@ -108,19 +124,235 @@ module NcsNavigator::Warehouse::Transformers
       end
     end
 
-    def interim_unsatisfied
-      @interim_unsatisfied ||= []
+    ##
+    # @private
+    class TransformTracker
+      def initialize(transform_status)
+        @transform_status = transform_status
+        @deferred_records = []
+      end
+
+      def defer(deferred_record)
+        @deferred_records << deferred_record
+      end
+
+      def end_it(fk_index)
+        update_satisfied_by_for_deferred(fk_index)
+        build_deferred_graph
+        compute_record_resolvabilities
+
+        deferred_satisfied, deferred_unsatisfied = partition_deferred
+
+        deferred_unsatisfied.each { |deferred_record| deferred_record.report_errors(@transform_status) }
+
+        deferred_satisfied.collect(&:record)
+      end
+
+      private
+
+      def update_satisfied_by_for_deferred(fk_index)
+        @deferred_records.each do |deferred_record|
+          deferred_record.update_satisfied_by_for_relationships(fk_index, @deferred_records)
+        end
+      end
+
+      def build_deferred_graph
+        require 'rgl/base'
+        require 'rgl/adjacency'
+        require 'rgl/connected_components'
+        require 'rgl/condensation'
+
+        @graph = RGL::DirectedAdjacencyGraph.new
+        @deferred_records.each do |rec|
+          rec.deferred_relationships.each do |rel|
+            @graph.add_edge(rec, rel.satisfied_by)
+          end
+        end
+      end
+
+      def compute_record_resolvabilities
+        # scc == strongly connected component; i.e., a single record or a complete cycle
+        @graph.condensation_graph.depth_first_search do |scc|
+          # skip the special terminal nodes :unsatisfed and :already_saved
+          next if Symbol === scc.first
+
+          # find all the external deferred relationships for the SCC
+          external_relationships = scc.collect { |rec|
+            rec.deferred_relationships.reject { |dr| scc.include?(dr.satisfied_by) }
+          }.flatten
+
+          # combine the resolvabilities for those those to determine the resolvability for the SCC
+          net_resolvability = external_relationships.collect(&:resolvable?).
+            reject { |r| r }.empty? # are there any false ones?
+
+          # update each DeferredRecord with the determined resolvability for the SCC
+          scc.each do |rec|
+            rec.resolvability = net_resolvability
+          end
+        end
+      end
+
+      def partition_deferred
+        @deferred_records.partition { |rec| rec.resolvable? }
+      end
+    end
+
+    ##
+    # @private
+    #
+    # These track records which are pending until the end of the transform
+    # and then are used as vertices in the graph which is used for analyzing
+    # remaining unsatisfied relationships.
+    class DeferredRecord
+      attr_reader :record, :deferred_relationships
+
+      ##
+      # The externally determined total resolvability for this record.
+      # @see TransformTracker#compute_record_resolvabilities
+      attr_writer :resolvability
+
+      ##
+      # @return [DeferredRecord,nil] if the record has any currently unsatisfied
+      #   FKs, return a properly initialized DeferredRecord.
+      def self.create_if_appropriate(record, fk_index)
+        relationships = record.class.relationships.collect { |belongs_to|
+          DeferredRelationship.create_if_appropriate(record, belongs_to, fk_index)
+        }.compact
+        if relationships.empty?
+          nil
+        else
+          DeferredRecord.new(record, relationships)
+        end
+      end
+
+      def initialize(record, deferred_relationships)
+        @record = record
+        @deferred_relationships = deferred_relationships
+      end
+
+      def model_class
+        record.class.to_s
+      end
+
+      def record_id
+        record.key.first
+      end
+
+      def update_satisfied_by_for_relationships(fk_index, deferred_pool)
+        deferred_relationships.each do |rel|
+          rel.update_satisfied_by(fk_index, deferred_pool)
+        end
+      end
+
+      def resolvability_determined?
+        !@resolvability.nil?
+      end
+
+      def resolvable?
+        unless resolvability_determined?
+          fail "Graph iteration failure: DR #{self} #resolvable? called before resolvability determined."
+        end
+        @resolvability
+      end
+
+      def report_errors(transform_status)
+        deferred_relationships.collect { |rel| rel.create_error }.compact.each do |error|
+          transform_status.transform_errors << error
+        end
+      end
+
+      def to_s
+        "#{model_class}##{record_id}"
+      end
     end
 
     ##
     # @private
-    class RelationshipInstance < Struct.new(:record_id, :model_class, :foreign_model, :reference_key, :reference_value)
+    class DeferredRelationship < Struct.new(:record, :foreign_model, :reference_key, :reference_value)
+      def self.create_if_appropriate(record, belongs_to, fk_index)
+        reference_name  = belongs_to.child_key.first.name
+        reference_value = record.send(reference_name)
+        foreign_model   = belongs_to.parent_model
+
+        if reference_value && !fk_index.seen?(foreign_model, reference_value)
+          DeferredRelationship.new(
+            record, foreign_model, reference_name, reference_value
+          )
+        else
+          nil
+        end
+      end
+
+      attr_accessor :satisfied_by
+
+      def initialize(*)
+        super
+        @satisfied_by = :unsatisfied
+      end
+
+      def record_id
+        record.key.first # No CPKs in MDES
+      end
+
+      def model_class
+        record.class.to_s
+      end
+
+      def update_satisfied_by(fk_index, deferred_pool)
+        if seen_in?(fk_index)
+          self.satisfied_by = :already_saved
+        elsif satisfier = deferred_pool.find { |deferred_record| self.satisfied_by_deferred?(deferred_record) }
+          self.satisfied_by = satisfier
+        end
+      end
+
+      def seen_in?(fk_index)
+        fk_index.seen?(foreign_model, reference_value)
+      end
+
+      def satisfied_by_deferred?(deferred_record)
+        self.foreign_model.to_s == deferred_record.model_class.to_s &&
+        self.reference_value == deferred_record.record_id
+      end
+
       def create_error
+        return nil if resolvable?
+
+        message =
+          case satisfied_by
+          when Symbol # i.e., :unsatisfied
+            "Unsatisfied foreign key referencing #{foreign_model}."
+          else
+            "Associated #{foreign_model} record contains one or more unsatisfed foreign keys or refers to other records that do."
+          end
+
         NcsNavigator::Warehouse::TransformError.new(
           :record_id => record_id, :model_class => model_class,
-          :message => "Unsatisfied foreign key #{reference_key}=#{reference_value} referencing #{foreign_model}."
+          :attribute_name => reference_key, :attribute_value => reference_value.inspect,
+          :message => message
         )
       end
+
+      def resolvable?
+        case satisfied_by
+        when :unsatisfied
+          false
+        when :already_saved
+          true
+        else
+          if satisfied_by.resolvability_determined?
+            satisfied_by.resolvable?
+          else
+            # The DFS over the coalesced graph should prevent this from happening,
+            # so this is a check against a programming fault or misconception.
+            fail "Graph iteration failure: FK #{self} #resolvable? called before satisfying-record resolvability determined."
+          end
+        end
+      end
+
+      def to_s
+        "#{model_class}##{record_id}.#{reference_key} => #{foreign_model}##{reference_value}"
+      end
     end
   end
 end