ncs_mdes_warehouse 0.4.1 → 0.5.0

data/CHANGELOG.md

@@ -1,6 +1,33 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.5.0
+-----
+
+- Add "post-ETL hooks" to ETL process: objects with callbacks which
+  are executed when the ETL completes. (#1725)
+
+- Add post-ETL hook for sending e-mail when the ETL process completes,
+  indicating success or failure. (#1601)
+
+- Include the input filename in the name of transformers based on
+  `VdrXml::Reader`. (#1927)
+
+- Exclude parent bundler environment when executing subprocess in
+  SubprocessTransformer. (#2012)
+
+- Strip leading and trailing whitespace from values in one-to-one
+  transformer. (#2028)
+
+- Catch all exceptions during enumeration in EnumTransformer. (#2070)
+
+- Log caught exceptions during ETL. Previously they were only reported
+  to the shell and stored in the transform status table. (#2070)
+
+- An enumerator may communicate recoverable errors to EnumTransformer
+  by yielding one or more TransformErrors as part of its
+  enumeration. (#2073)
+
 0.4.1
 -----
 

data/Rakefile

@@ -14,12 +14,12 @@ task :spec => 'spec:all'
 
 namespace :spec do
   RSpec::Core::RakeTask.new(:fast) do |t|
-    t.pattern = "spec/**/*_spec.rb"
+    t.pattern = ENV['SPEC_PATTERN'] || "spec/**/*_spec.rb"
     t.rspec_opts = %q(--tag ~slow)
   end
 
   RSpec::Core::RakeTask.new(:all) do |t|
-    t.pattern = "spec/**/*_spec.rb"
+    t.pattern = ENV['SPEC_PATTERN'] || "spec/**/*_spec.rb"
   end
 end
 

data/lib/ncs_navigator/warehouse.rb

@@ -11,8 +11,10 @@ module NcsNavigator
     autoload :Configuration,       'ncs_navigator/warehouse/configuration'
     autoload :DataMapper,          'ncs_navigator/warehouse/data_mapper'
     autoload :DatabaseInitializer, 'ncs_navigator/warehouse/database_initializer'
+    autoload :Hooks,               'ncs_navigator/warehouse/hooks'
     autoload :Models,              'ncs_navigator/warehouse/models'
     autoload :PostgreSQL,          'ncs_navigator/warehouse/postgresql'
+    autoload :StringifyTrace,      'ncs_navigator/warehouse/stringify_trace'
     autoload :TableModeler,        'ncs_navigator/warehouse/table_modeler'
     autoload :Transformers,        'ncs_navigator/warehouse/transformers'
     autoload :TransformError,      'ncs_navigator/warehouse/transform_status'

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -82,6 +82,36 @@ module NcsNavigator::Warehouse
       end
     end
 
+    ####
+    #### Hooks
+    ####
+
+    ##
+    # @return [Array<#etl_succeeded,#etl_failed>] the configured
+    #   post-ETL hooks.
+    def post_etl_hooks
+      @post_etl_hooks ||= []
+    end
+
+    ##
+    # Adds a post-ETL hook to the list for this warehouse instance.
+    #
+    # @return [void]
+    # @param [#etl_succeeded,#etl_failed] the hook
+    def add_post_etl_hook(candidate)
+      expected_methods = [:etl_succeeded, :etl_failed]
+      implemented_methods = expected_methods.select { |m| candidate.respond_to?(m) }
+      if implemented_methods.empty?
+        msg = "#{candidate.inspect} does not have an #{expected_methods.join(' or ')} method."
+        if candidate.respond_to?(:new)
+          msg += " Perhaps you meant #{candidate}.new?"
+        end
+        raise Error, msg
+      else
+        post_etl_hooks << candidate
+      end
+    end
+
     ####
     #### MDES version
     ####
@@ -157,6 +187,26 @@ module NcsNavigator::Warehouse
       @navigator = NcsNavigator::Configuration.new(ini_file)
     end
 
+    ###
+    ### E-mail
+    ###
+
+    ##
+    # Configures `ActionMailer` with the options implied by the suite
+    # configuration.
+    #
+    # @return [void]
+    def set_up_action_mailer
+      return if @action_mailer_set_up
+      require 'action_mailer'
+      ActionMailer::Base.delivery_method = :smtp
+      ActionMailer::Base.smtp_settings = navigator.action_mailer_smtp_settings
+      ActionMailer::Base.view_paths = [
+        File.expand_path('../mailer_templates', __FILE__)
+      ]
+      @action_mailer_set_up = true
+    end
+
     ####
     #### Bcdatabase
     ####

data/lib/ncs_navigator/warehouse/hooks.rb

@@ -0,0 +1,21 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse
+  ##
+  # The namespace for post-ETL hook implementations which are provided
+  # with the warehouse.
+  #
+  # A post-ETL hook is an object which responds to either
+  # `etl_succeeded` or `etl_failed` (or both). Each method takes a
+  # single hash argument which, when the method is called, will
+  # contain the following keys:
+  #
+  # * `:transform_statuses` the list of {TransformStatus}es describing
+  #   the ETL process that was just completed
+  # * `:configuration` a reference to the warehouse {Configuration}.
+  #
+  # @see Configuration#add_post_etl_hook
+  module Hooks
+    autoload :EtlStatusEmail, 'ncs_navigator/warehouse/hooks/etl_status_email'
+  end
+end

data/lib/ncs_navigator/warehouse/hooks/etl_status_email.rb

@@ -0,0 +1,88 @@
+require 'ncs_navigator/warehouse'
+require 'action_mailer'
+require 'time'
+
+module NcsNavigator::Warehouse::Hooks
+  ##
+  # A post-ETL hook which sends a message to configured e-mail
+  # accounts after each ETL run. The message indicates whether the ETL
+  # succeeded or failed and some summary statistics, but no further
+  # detail.
+  class EtlStatusEmail
+    ##
+    # @param [Hash<Symbol, Object>] options
+    # @option options [Array<String>] :to the e-mail addresses to
+    #   whom the notifications will be sent.
+    def initialize(options={})
+      @to = options[:to] or 'Need at least one recipient'
+    end
+
+    ##
+    # @param [Hash<Symbol, Object>] args the arguments received from
+    #   the ETL process.
+    # @return [void]
+    def etl_succeeded(args)
+      args[:configuration].set_up_action_mailer
+
+      Mailer.success_message(@to, args[:transform_statuses]).deliver
+    end
+
+    ##
+    # @param [Hash<Symbol, Object>] args the arguments received from
+    #   the ETL process.
+    # @return [void]
+    def etl_failed(args)
+      args[:configuration].set_up_action_mailer
+
+      Mailer.failure_message(@to, args[:transform_statuses]).deliver
+    end
+
+    ##
+    # @private
+    class Mailer < ::ActionMailer::Base
+      self.mailer_name = 'etl_status_email'
+
+      def success_message(to, transform_statuses)
+        analyze_statuses(transform_statuses)
+
+        mail(
+          # TODO: make configurable
+          :from => 'mdes-warehouse',
+          :to => to,
+          :subject => '[NCS Navigator] Warehouse load successful'
+        )
+      end
+
+      def failure_message(to, transform_statuses)
+        analyze_statuses(transform_statuses)
+
+        mail(
+          # TODO: make configurable
+          :from => 'mdes-warehouse',
+          :to => to,
+          :subject => '[NCS Navigator] Warehouse load failed'
+        )
+      end
+
+      private
+
+      def analyze_statuses(transform_statuses)
+        start_time_dt = transform_statuses.first.start_time
+        end_time_dt = transform_statuses.last.end_time
+
+        @start_time = start_time_dt.to_s
+
+        @transform_duration = duration_string((end_time_dt - start_time_dt) * 24 * 3600)
+        @transform_count = transform_statuses.size
+        @record_count = transform_statuses.inject(0) { |sum, s| sum + s.record_count }
+
+        @success_count = transform_statuses.select { |s| s.transform_errors.empty? }.size
+        @failure_count = transform_statuses.size - @success_count
+      end
+
+      def duration_string(seconds)
+        [ seconds / 3600, (seconds % 3600) / 60, seconds % 60 ].collect { |t| '%02d' % t }.join(':')
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/mailer_templates/etl_status_email/failure_message.text.erb

@@ -0,0 +1,8 @@
+An MDES Warehouse transform and load process ran, starting at <%= @start_time %>.
+
+<%= @success_count %> transformation<%= 's' unless @success_count == 1 %> executed successfully.
+<%= @failure_count %> transformation<%= 's' unless @failure_count == 1 %> failed.
+
+Executing <%= @transform_count %> transform<%= 's' if @transform_count != 1 %>
+took <%= @transform_duration %> and
+produced <%= @record_count %> total record<%= 's' if @record_count != 1 %>.

data/lib/ncs_navigator/warehouse/mailer_templates/etl_status_email/success_message.text.erb

@@ -0,0 +1,7 @@
+An MDES Warehouse transform and load process ran, starting at <%= @start_time %>.
+
+All transformations executed successfully.
+
+Executing <%= @transform_count %> transform<%= 's' if @transform_count != 1 %>
+took <%= @transform_duration %> and
+produced <%= @record_count %> total record<%= 's' if @record_count != 1 %>.

data/lib/ncs_navigator/warehouse/stringify_trace.rb

@@ -0,0 +1,31 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse
+  ##
+  # @api private
+  module StringifyTrace
+    ##
+    # Utility to generate a nicely formatted string from a ruby
+    # exception trace.
+    #
+    # @param [Array<String>] backtrace
+    # @return String
+    def stringify_trace(backtrace)
+      # an array of arrays containing [filename, line, msg]
+      trace_lines = backtrace.collect { |l| l.scan(/^(.*?)\:(\d*)\:?(.*)$/).first || ['', '', l] }
+
+      lengths = trace_lines.inject([0, 0, 0]) { |lens, components|
+        0.upto(2) { |i| lens[i] = [lens[i], components[i].length].max }
+        lens
+      }
+
+      formats = ["%#{lengths[0]}s", "%#{lengths[1]}s", "%s"]
+      trace_lines.collect { |components|
+        formats.zip(components).collect { |format, component|
+          (format % component)
+        }.select { |s| s =~ /\S/ }.join(':')
+      }.join("\n")
+    end
+    module_function :stringify_trace
+  end
+end

data/lib/ncs_navigator/warehouse/transform_load.rb

@@ -5,6 +5,7 @@ require 'forwardable'
 module NcsNavigator::Warehouse
   class TransformLoad
     extend Forwardable
+    include StringifyTrace
 
     attr_reader :configuration
     attr_reader :statuses
@@ -34,12 +35,16 @@ module NcsNavigator::Warehouse
                   transformer.transform(status)
                 rescue => e
                   shell.say_line("\nTransform failed. (See log for more detail.)")
-                  status.add_error("Transform failed. #{e.class}: #{e}.")
+                  msg = "Transform failed. #{e.class}: #{e}\n#{stringify_trace(e.backtrace)}"
+                  log.error(msg)
+                  status.add_error(msg)
                 end
               end
             rescue DataObjects::IntegrityError => e
-              shell.say_line("\nTransform failed with data integrity error. (See log for more detail.)")
-              log.error("Transform failed with data integrity error: #{e}.")
+              shell.say_line(
+                "\nTransform failed with data integrity error. (See log for more detail.)")
+              log.error(
+                "Transform failed with data integrity error: #{e}.\n#{stringify_trace(e.backtrace)}")
               status.add_error("Transform failed with data integrity error: #{e}.")
             end
             status.end_time = Time.now
@@ -53,8 +58,10 @@ module NcsNavigator::Warehouse
       end
 
       if statuses.detect { |s| !s.transform_errors.empty? }
+        dispatch_post_etl_hooks(:etl_failed)
         false
       else
+        dispatch_post_etl_hooks(:etl_succeeded)
         true
       end
     end
@@ -67,5 +74,18 @@ module NcsNavigator::Warehouse
         )
     end
     private :build_status_for
+
+    def dispatch_post_etl_hooks(method)
+      configuration.post_etl_hooks.each do |hook|
+        begin
+          args = { :transform_statuses => statuses, :configuration => configuration }
+          hook.send(method, args) if hook.respond_to?(method)
+        rescue => e
+          log.error(
+            "Error invoking #{method.inspect} on #{hook.inspect}: #{e.class} #{e}.\n#{stringify_trace(e.backtrace)}")
+        end
+      end
+    end
+    private :dispatch_post_etl_hooks
   end
 end

data/lib/ncs_navigator/warehouse/transform_status.rb

@@ -14,8 +14,8 @@ module NcsNavigator::Warehouse
     # tests). This method creates a new instance which works around
     # this problem, at the cost of the instances not being accurately
     # persistable.
-    def self.memory_only(name)
-      TransformStatus.new(:name => name).tap do |s|
+    def self.memory_only(name, attrs={})
+      TransformStatus.new(attrs.merge(:name => name)).tap do |s|
         def s.transform_errors
           @transform_errors ||= []
         end
@@ -59,6 +59,15 @@ module NcsNavigator::Warehouse
     property :record_id,   String, :length => 255
 
     belongs_to :transform_status, TransformStatus, :required => true
+
+    def self.for_exception(exception, context_message=nil)
+      TransformError.new(:message => [
+          context_message,
+          "#{exception.class}: #{exception}",
+          StringifyTrace.stringify_trace(exception.backtrace)
+        ].compact.join("\n")
+      )
+    end
   end
 
   TransformError.finalize

data/lib/ncs_navigator/warehouse/transformers/database.rb

@@ -327,11 +327,23 @@ module NcsNavigator::Warehouse::Transformers
           raise UnusedColumnsForModelError.new(unused)
         end
         model.new(
-          col_map.inject({}) { |pv, (col_name, var_name)| pv[var_name] = row[col_name]; pv }
+          col_map.inject({}) { |pv, (col_name, var_name)|
+            pv[var_name] = clean_value(row[col_name]);
+            pv
+          }
         )
       end
       alias :call :convert_row
 
+      def clean_value(v)
+        if v.respond_to?(:strip)
+          v.strip
+        else
+          v
+        end
+      end
+      private :clean_value
+
       ##
       # @param [Array<String>] column_names
       # @return [Hash<String, String>] a mapping from the given

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -4,12 +4,28 @@ require 'forwardable'
 
 module NcsNavigator::Warehouse::Transformers
   ##
-  # A transformer that accepts a series of model instances in the form
-  # of a ruby Enumerable. An enumerable might be as simple as an
-  # array, or it might be a custom class that streams through
-  # thousands of instances without having them all in memory at once.
+  # A transformer that accepts a series of model instances and
+  # {TransformError}s in the form of a ruby Enumerable. An enumerable
+  # might be as simple as an array, or it might be a custom class that
+  # streams through thousands of instances without having them all in
+  # memory at once.
+  #
+  # Each value yielded by the enumerable may be either an instance of
+  # an MDES model or a {TransformError}. If it is a model instance, it
+  # will have global values (e.g., PSU ID) applied as necessary,
+  # validated, and saved.
+  #
+  # On the other hand, If it is a `TransformError` the error will be
+  # associated with the status for the transform run. The benefit of
+  # the enumeration yielding a `TransformError` instead of throwing an
+  # exception is that the enumeration may continue after the error is
+  # reported. If the error is unrecoverable, the enum should throw an
+  # exception instead of returning a
+  # `TransformError`. `EnumTransformer` will handle recording the
+  # error appropriately in that case.
   class EnumTransformer
     extend Forwardable
+    include NcsNavigator::Warehouse::StringifyTrace
 
     ##
     # @return [Enumerable] the enumeration that will be transformed.
@@ -26,7 +42,12 @@ module NcsNavigator::Warehouse::Transformers
     end
 
     def name
-      "EnumTransformer for #{enum.class}"
+      enum_name = if enum.respond_to?(:name)
+                    enum.name
+                  else
+                    enum.class
+                  end
+      "EnumTransformer for #{enum_name}"
     end
 
     ##
@@ -36,31 +57,55 @@ module NcsNavigator::Warehouse::Transformers
     # @param [TransformStatus] status
     # @return [void]
     def transform(status)
+      begin
+        do_transform(status)
+      rescue Exception => e
+        err = NcsNavigator::Warehouse::TransformError.for_exception(e, 'Enumeration failed.')
+        log.error err.message
+        status.transform_errors << err
+      end
+    end
+
+    private
+
+    def do_transform(status)
       enum.each do |record|
-        apply_global_values_if_necessary(record)
-        if record.valid?
-          log.debug("Saving valid record #{record_ident record}.")
-          begin
-            unless record.save
-              msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
-              log.error msg
-              status.unsuccessful_record(record, msg)
-            end
-          rescue => e
-            msg = "Error on save. #{e.class}: #{e}."
+        case record
+        when NcsNavigator::Warehouse::TransformError
+          receive_transform_error(record, status)
+        else
+          save_model_instance(record, status)
+        end
+      end
+    end
+
+    def save_model_instance(record, status)
+      apply_global_values_if_necessary(record)
+      if record.valid?
+        log.debug("Saving valid record #{record_ident record}.")
+        begin
+          unless record.save
+            msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
             log.error msg
             status.unsuccessful_record(record, msg)
           end
-        else
-          msg = "Invalid record. #{record_messages(record).join(' ')}"
+        rescue => e
+          msg = "Error on save. #{e.class}: #{e}."
           log.error msg
           status.unsuccessful_record(record, msg)
         end
-        status.record_count += 1
+      else
+        msg = "Invalid record. #{record_messages(record).join(' ')}"
+        log.error msg
+        status.unsuccessful_record(record, msg)
       end
+      status.record_count += 1
     end
 
-    private
+    def receive_transform_error(error, status)
+      error.id = nil
+      status.transform_errors << error
+    end
 
     def record_ident(rec)
       # No composite keys in the MDES