ncs_mdes_warehouse 0.6.2 → 0.7.0

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -82,6 +82,27 @@ module NcsNavigator::Warehouse
       end
     end
 
+    ##
+    # The foreign key index used during an ETL run. The default value
+    # is correct for virtually any case.
+    #
+    # @return [Object]
+    def foreign_key_index
+      @foreign_key_index ||= Transformers::ForeignKeyIndex.new
+    end
+
+    ##
+    # Specify a different foreign key index implementation to
+    # use. This will only rarely be necessary or useful. The default
+    # value is correct for virtually any case.
+    #
+    # @return [void]
+    # @param [#record_and_verify,#report_errors] index the replacement
+    #   foreign key index implementation.
+    def foreign_key_index=(index)
+      @foreign_key_index = index
+    end
+
     ####
     #### Hooks
     ####

data/lib/ncs_navigator/warehouse/transformers.rb

@@ -17,10 +17,12 @@ module NcsNavigator::Warehouse
     autoload :EnumTransformer,         'ncs_navigator/warehouse/transformers/enum_transformer'
     autoload :EventStartFromContactTransformer, 'ncs_navigator/warehouse/transformers/event_start_from_contact_transformer'
     autoload :Filters,                 'ncs_navigator/warehouse/transformers/filters'
+    autoload :ForeignKeyIndex,         'ncs_navigator/warehouse/transformers/foreign_key_index'
     autoload :NoBlankForeignKeysFilter,       'ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter'
     autoload :NoSsuOutreachAllSsusFilter,     'ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter'
     autoload :NoSsuOutreachPlaceholderFilter, 'ncs_navigator/warehouse/transformers/no_ssu_outreach_placeholder_filter'
     autoload :SamplingUnits,           'ncs_navigator/warehouse/transformers/sampling_units'
+    autoload :SqlTransformer,          'ncs_navigator/warehouse/transformers/sql_transformer'
     autoload :SubprocessTransformer,   'ncs_navigator/warehouse/transformers/subprocess_transformer'
     autoload :VdrXml,                  'ncs_navigator/warehouse/transformers/vdr_xml'
   end

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -35,7 +35,12 @@ module NcsNavigator::Warehouse::Transformers
     # @return [Filters] the filters in use on this transformer.
     attr_reader :filters
 
-    def_delegators :@configuration, :log, :shell
+    ##
+    # @return [Symbol] the name of the duplicates mode in use on this
+    # transformer.
+    attr_reader :duplicates
+
+    def_delegators :@configuration, :log, :shell, :foreign_key_index
 
     ##
     # @param [Configuration] configuration
@@ -43,6 +48,11 @@ module NcsNavigator::Warehouse::Transformers
     # @param [Hash<Symbol, Object>] options
     # @option options [Array<#call>,#call] :filters a list of
     #   filters to use for this transformer
+    # @option options [:error,:ignore,:replace] the desired behavior when this
+    #   transformer encounters a record with the same PK as one already seen.
+    #   `:error` means blindly save (and let the database error out).
+    #   `:ignore` means do not attempt to save the duplicate.
+    #   `:replace` means substitute the duplicate for the existing record.
     #
     # @see Filters
     def initialize(configuration, enum, options={})
@@ -50,6 +60,8 @@ module NcsNavigator::Warehouse::Transformers
       @enum = enum
       filter_list = options.delete(:filters)
       @filters = Filters.new(filter_list ? [*filter_list].compact : [])
+      @duplicates = options.delete(:duplicates) || :error
+      @duplicates_strategy = select_duplicates_strategy
     end
 
     ##
@@ -90,45 +102,62 @@ module NcsNavigator::Warehouse::Transformers
           receive_transform_error(record, status)
         else
           filters.call([record]).each do |filtered_record|
-            save_model_instance(filtered_record, status)
+            saved_record = save_model_instance(filtered_record, status)
+            foreign_key_index.record_and_verify(saved_record) if saved_record
           end
         end
       end
+      foreign_key_index.report_errors(status)
     end
 
-    def save_model_instance(record, status)
-      if !has_valid_psu?(record)
-        msg = "Invalid PSU ID. The list of valid PSU IDs for this Study Center is #{@configuration.navigator.psus.collect(&:id).inspect}."
-        log.error "#{record_ident record}: #{msg}"
-        status.unsuccessful_record(record, msg,
-          :attribute_name => 'psu_id',
-          :attribute_value => record.psu_id.inspect)
-      elsif record.valid?
-        log.debug("Saving valid record #{record_ident record}.")
-        begin
-          unless record.save
-            msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
+    def save_model_instance(incoming_record, status)
+      record = process_duplicate_if_appropriate(incoming_record)
+      unless record
+        log.info("Ignoring duplicate record #{record_ident incoming_record}.")
+        status.record_count += 1
+        return
+      end
+
+      saved_record =
+        if !has_valid_psu?(record)
+          msg = "Invalid PSU ID. The list of valid PSU IDs for this Study Center is #{@configuration.navigator.psus.collect(&:id).inspect}."
+          log.error "#{record_ident record}: #{msg}"
+          status.unsuccessful_record(record, msg,
+            :attribute_name => 'psu_id',
+            :attribute_value => record.psu_id.inspect)
+          nil
+        elsif record.valid?
+          log.debug("Saving valid record #{record_ident record}.")
+          begin
+            if record.save
+              record
+            else
+              msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
+              log.error msg
+              status.unsuccessful_record(record, msg)
+              nil
+            end
+          rescue => e
+            msg = "Error on save. #{e.class}: #{e}."
             log.error msg
             status.unsuccessful_record(record, msg)
+            nil
           end
-        rescue => e
-          msg = "Error on save. #{e.class}: #{e}."
-          log.error msg
-          status.unsuccessful_record(record, msg)
-        end
-      else
-        log.error "Invalid record. #{record_messages(record).join(' ')}"
-        record.errors.keys.each do |prop|
-          record.errors[prop].each do |e|
-            status.unsuccessful_record(
-              record, "Invalid: #{e}.",
-              :attribute_name => prop,
-              :attribute_value => record.send(prop).inspect
-            )
+        else
+          log.error "Invalid record. #{record_messages(record).join(' ')}"
+          record.errors.keys.each do |prop|
+            record.errors[prop].each do |e|
+              status.unsuccessful_record(
+                record, "Invalid: #{e}.",
+                :attribute_name => prop,
+                :attribute_value => record.send(prop).inspect
+              )
+            end
           end
+          nil
         end
-      end
       status.record_count += 1
+      saved_record
     end
 
     def receive_transform_error(error, status)
@@ -163,5 +192,77 @@ module NcsNavigator::Warehouse::Transformers
         true
       end
     end
+
+    def process_duplicate_if_appropriate(record)
+      if @duplicates_strategy.duplicate?(record)
+        @duplicates_strategy.to_save(record)
+      else
+        record
+      end
+    end
+
+    def select_duplicates_strategy
+      case duplicates
+      when :error
+        ErrorDuplicatesStrategy.new
+      when :ignore
+        IgnoreDuplicatesStrategy.new(@configuration)
+      when :replace
+        ReplaceDuplicatesStrategy.new(@configuration)
+      else
+        fail "Unknown duplicates mode #{duplicates.inspect}."
+      end
+    end
+
+    ##
+    # @private
+    class ErrorDuplicatesStrategy
+      def duplicate?(record)
+        false
+      end
+
+      # to_save will never be called for this strategy
+    end
+
+    ##
+    # @private
+    class AbstractDoSomethingWithDuplicatesStrategy
+      extend Forwardable
+
+      def_delegators :@configuration, :log
+
+      def initialize(configuration)
+        @configuration = configuration
+        @fk_index = configuration.foreign_key_index
+      end
+
+      def duplicate?(record)
+        @fk_index.seen?(record.class, record.key.first)
+      end
+    end
+
+    ##
+    # @private
+    class IgnoreDuplicatesStrategy < AbstractDoSomethingWithDuplicatesStrategy
+      def to_save(duplicate_record)
+        nil
+      end
+    end
+
+    ##
+    # @private
+    class ReplaceDuplicatesStrategy < AbstractDoSomethingWithDuplicatesStrategy
+      def to_save(duplicate_record)
+        log.info "Updating duplicate record #{duplicate_record.class}##{duplicate_record.key.first}"
+
+        reloaded = duplicate_record.class.get(*duplicate_record.key)
+
+        duplicate_record.class.properties.each do |prop|
+          reloaded[prop.name] = duplicate_record[prop.name]
+        end
+
+        reloaded
+      end
+    end
   end
 end

data/lib/ncs_navigator/warehouse/transformers/foreign_key_index.rb

@@ -0,0 +1,126 @@
+require 'ncs_navigator/warehouse'
+require 'set'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # Provides for in-memory reporting of foreign key violations during
+  # ETL runs.
+  #
+  # When FK constraint violations are delegated to the database, only
+  # one can be reported per ETL execution. This class (or one like it)
+  # allows for all foreign key violations to be reported across an
+  # entire run.
+  #
+  # @see Configuration#foreign_key_index
+  class ForeignKeyIndex
+    autoload :DatabaseKeyProvider, 'ncs_navigator/warehouse/transformers/foreign_key_index/database_key_provider'
+    autoload :StaticKeyProvider,   'ncs_navigator/warehouse/transformers/foreign_key_index/static_key_provider'
+
+    ##
+    # The object that will be used to pre-initialize the known keys list for
+    # a particular model. It has a single method (`existing_keys`) which takes
+    # a single argument, the model class for which keys should be provided, and
+    # returns an array of the known keys for that model.
+    #
+    # `existing_keys` will be called at most once for each model class. It will
+    # be called the first time either a record of the model's type is
+    # encountered or the first time a foreign key of the model's type is
+    # encountered.
+    #
+    # @return [#existing_keys]
+    attr_reader :existing_key_provider
+
+    ##
+    # @param [Hash<Symbol, Object>] options
+    # @option options [#existing_keys] :existing_key_provider (an instance of DatabaseKeyProvider)
+    #   See {#existing_key_provider} for more information.
+    def initialize(options={})
+      @existing_key_provider = if options.has_key?(:existing_key_provider)
+        options[:existing_key_provider] || StaticKeyProvider.new # empty if nil
+      else
+        DatabaseKeyProvider.new
+      end
+      @seen_keys = {}
+    end
+
+    ##
+    # Records the key for this record in the index and verifies its
+    # foreign keys. If any are not satisfied, they will be stored for
+    # later evaluation.
+    #
+    # @param [DataMapper::Resource] record the record whose key we
+    #   want to record and whose foreign references we want to verify.
+    # @return [void]
+    def record_and_verify(record)
+      seen_keys(record.class) << record.key.first # no CPKs in MDES
+
+      record.class.relationships.each do |belongs_to|
+        verify_relationship(record, belongs_to)
+      end
+    end
+
+    ##
+    # Reviews any references that initially failed against the final
+    # set of keys. If any are still unresolveable, it records errors
+    # against the provided transform status.
+    #
+    # Each failed reference will be reported by this method only
+    # once. This is so that it is safe to share a single instance of
+    # this class across multiple transformers, reporting only the
+    # newly encountered errors for each transform in turn.
+    #
+    # @return [void]
+    # @param [TransformStatus] transform_status
+    def report_errors(transform_status)
+      interim_unsatisfied.each do |relationship_instance|
+        if !seen?(relationship_instance.foreign_model, relationship_instance.reference_value)
+          transform_status.transform_errors << relationship_instance.create_error
+        end
+      end
+      interim_unsatisfied.clear
+    end
+
+    ##
+    # @return [Boolean] has the index seen a record of the given type with the
+    #   given ID.
+    def seen?(model_class, id)
+      seen_keys(model_class).include?(id)
+    end
+
+    private
+
+    def verify_relationship(record, belongs_to)
+      reference_name  = belongs_to.child_key.first.name
+      reference_value = record.send(reference_name)
+      foreign_model   = belongs_to.parent_model.to_s
+
+      if reference_value && !seen?(foreign_model, reference_value)
+        interim_unsatisfied << RelationshipInstance.new(
+          record.id, record.class.to_s, foreign_model, reference_name, reference_value
+        )
+      end
+    end
+
+    def seen_keys(model_class)
+      @seen_keys[model_class.to_s] ||= begin
+        existing_keys = existing_key_provider.existing_keys(model_class) || []
+        Set.new(existing_keys)
+      end
+    end
+
+    def interim_unsatisfied
+      @interim_unsatisfied ||= []
+    end
+
+    ##
+    # @private
+    class RelationshipInstance < Struct.new(:record_id, :model_class, :foreign_model, :reference_key, :reference_value)
+      def create_error
+        NcsNavigator::Warehouse::TransformError.new(
+          :record_id => record_id, :model_class => model_class,
+          :message => "Unsatisfied foreign key #{reference_key}=#{reference_value} referencing #{foreign_model}."
+        )
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/foreign_key_index/database_key_provider.rb

@@ -0,0 +1,16 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers
+  class ForeignKeyIndex
+    ##
+    # Provides existing key lists out of the configured DataMapper "working"
+    # repo.
+    class DatabaseKeyProvider
+      def existing_keys(model_class)
+        ::DataMapper.repository(:mdes_warehouse_working).adapter.select(
+          "SELECT #{model_class.key.first.name} FROM #{model_class.mdes_table_name}"
+        )
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/foreign_key_index/static_key_provider.rb

@@ -0,0 +1,20 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers
+  class ForeignKeyIndex
+    ##
+    # An existing key provider for {ForeignKeyIndex} that uses a statically
+    # configured hash of keys.
+    class StaticKeyProvider
+      attr_accessor :known_keys
+
+      def initialize(known_keys={})
+        @known_keys = known_keys
+      end
+
+      def existing_keys(model_class)
+        known_keys[model_class.to_s]
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/sql/sql_statements.treetop

@@ -0,0 +1,27 @@
+module NcsNavigator::Warehouse::Transformers::Sql
+  grammar SqlStatements
+    rule script
+      stmts:(stmt:statement ';')* last_stmt:statement? {
+        def statements
+          (stmts + [last_stmt]).collect { |s| s.text_value.strip }.reject { |s| s.empty? }
+        end
+
+        def stmts
+          super.elements.map { |e| e.stmt }
+        end
+      }
+    end
+
+    rule statement
+      (sql_string / other_expression)+
+    end
+
+    rule other_expression
+      [^';]+
+    end
+
+    rule sql_string
+      "'" (!"'" .)* "'"
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/sql_transformer.rb

@@ -0,0 +1,107 @@
+require 'ncs_navigator/warehouse'
+
+require 'forwardable'
+require 'treetop'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # A transformer that executes raw SQL (most likely INSERTs or UPDATEs) on the
+  # warehouse working database during ETL. It captures and reports affected
+  # record counts and errors.
+  class SqlTransformer
+    class << self
+      ##
+      # Splits a string into separate SQL statements on semicolons. It attempts
+      # to avoid splitting on semicolons that are in string literals, though it
+      # only currently supports standard SQL string literals completely. It will
+      # not properly handle PostgreSQL `E''` string literals that contain
+      # backslash-escaped single quotes.
+      def extract_statements(s)
+        Treetop.load File.expand_path('../sql/sql_statements.treetop', __FILE__)
+
+        parser = Sql::SqlStatementsParser.new
+        if result = parser.parse(s)
+          result.statements
+        else
+          fail "Parse failed: #{parser.failure_reason}"
+        end
+      end
+    end
+
+    extend Forwardable
+
+    def_delegators :@configuration, :shell, :log
+    attr_reader :statements, :name
+
+    ##
+    # Creates a new SqlTransformer. You must provide at exactly one of the
+    # `:statements`, `:file`, or `:script` options. If you provide more than one
+    # the behavior of this transformer is undefined.
+    #
+    # @param [Configuration] configuration
+    # @param [Hash<Symbol, Object>] options
+    #
+    # @option options :statements [Array<String>] a list of SQL statements
+    #   already separated into strings. These strings will be sent as-is to the
+    #   database (no further processing).
+    # @option options :file [String] the name of a file from which to read
+    #   semicolon-separated SQL statements. The file will be separated into
+    #   statements using {.extract_statements}.
+    # @option options :script [String] a string containing semicolon-separated
+    #   SQL statements. The string will be separated into statements using
+    #   {.extract_statements}.
+    # @option options :name [String] the name to use when describing this
+    #   transformer. A default will be generated if not provided.
+    # @option options :adapter [a datamapper adapter object] the adapter
+    #   against which to `execute` the statements. The default is nearly always
+    #   fine; this option is provided for testing.
+    def initialize(configuration, options={})
+      @configuration = configuration
+      @statements = select_statements(options)
+      @name = options.delete(:name) ||
+        "SQL Transformer with #{statements.size} statement#{statements.size == 1 ? '' : 's'}"
+      @adapter = options.delete(:adapter) || ::DataMapper.repository(:mdes_warehouse_working).adapter
+    end
+
+    ##
+    # Executes the configured SQL statments, one at time, stopping at the first
+    # error.
+    #
+    # @return [void]
+    # @param [TransformStatus] transform_status
+    def transform(transform_status)
+      NcsNavigator::Warehouse::TransformStatus.transaction do |tx|
+        stmt_ct = statements.size
+        statements.each_with_index do |stmt, i|
+          log.debug("Executing SQL statement in SQL transformer: \n#{stmt}")
+          shell.clear_line_and_say("[#{name}] Executing statement #{i + 1}/#{stmt_ct}...")
+          begin
+            result = @adapter.execute(stmt)
+            transform_status.record_count += result.affected_rows
+          rescue Exception => e
+            transform_status.transform_errors << NcsNavigator::Warehouse::TransformError.
+              for_exception(e, "Exception while executing SQL statement \"#{stmt}\" (#{i + 1} of #{stmt_ct}).")
+            shell.clear_line_and_say("[#{name}] Failed on #{i + 1}/#{stmt_ct}.")
+            tx.rollback
+            return
+          end
+        end
+        shell.clear_line_and_say("[#{name}] Executed #{stmt_ct} SQL statement#{stmt_ct == 1 ? '' : 's'}.")
+      end
+    end
+
+    private
+
+    def select_statements(options)
+      if options[:statements]
+        options[:statements]
+      elsif options[:script]
+        self.class.extract_statements(options[:script])
+      elsif options[:file]
+        self.class.extract_statements(File.read(options[:file]))
+      else
+        fail "One of :file, :script, or :statements must be specified."
+      end
+    end
+  end
+end