ncs_mdes_warehouse 0.0.2 → 0.1.0

data/lib/ncs_navigator/warehouse/transform_load.rb

@@ -0,0 +1,55 @@
+require 'ncs_navigator/warehouse'
+
+require 'forwardable'
+
+module NcsNavigator::Warehouse
+  class TransformLoad
+    extend Forwardable
+
+    attr_reader :configuration
+    attr_reader :statuses
+
+    def_delegators :@configuration, :log, :shell
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    def run
+      position = 0
+      @statuses = configuration.transformers.collect do |transformer|
+        ::DataMapper.repository(:mdes_warehouse_working) do
+          build_status_for(transformer, position).tap do |status|
+            begin
+              transformer.transform(status)
+            rescue => e
+              shell.say_line("\nTransform failed. (See log for more detail.)")
+              status.add_error("Transform failed. #{e.class}: #{e}.")
+            end
+            status.end_time = Time.now
+            unless status.save
+              shell.say_line("Could not save status for transformer #{status.name}")
+              log.warn("Could not save status for transformer #{status.name}")
+            end
+            position += 1
+          end
+        end
+      end
+
+      if statuses.detect { |s| !s.transform_errors.empty? }
+        false
+      else
+        true
+      end
+    end
+
+    def build_status_for(transformer, position)
+      TransformStatus.new(
+        :name => transformer.respond_to?(:name) ? transformer.name : transformer.class.name,
+        :start_time => Time.now,
+        :position => position
+        )
+    end
+    private :build_status_for
+  end
+end

data/lib/ncs_navigator/warehouse/transform_status.rb

@@ -0,0 +1,63 @@
+require 'ncs_navigator/warehouse'
+
+require 'data_mapper'
+
+module NcsNavigator::Warehouse
+  ##
+  # Tracks and stores the progress of a particular transform.
+  class TransformStatus
+    include ::DataMapper::Resource
+
+    ##
+    # DataMapper 1.2 attempts to query for associations, even if the
+    # record has never been saved. This fails if the database
+    # connection is not set up (as in most of this library's
+    # tests). This method creates a new instance which works around
+    # this problem, at the cost of the instances not being accurately
+    # persistable.
+    def self.memory_only(name)
+      TransformStatus.new(:name => name).tap do |s|
+        def s.transform_errors
+          @transform_errors ||= []
+        end
+      end
+    end
+
+    storage_names[:default] = storage_names[:mdes_warehouse_working] =
+      storage_names[:mdes_warehouse_reporting] = 'wh_transform_status'
+
+    property :id,           Serial
+    property :name,         String,  :required => true, :length => 255
+    property :start_time,   DateTime
+    property :end_time,     DateTime
+    property :record_count, Integer, :default => 0
+    property :position,     Integer
+
+    has n, :transform_errors, 'NcsNavigator::Warehouse::TransformError'
+
+    def add_error(message)
+      self.transform_errors << TransformError.new(:message => message)
+    end
+
+    def unsuccessful_record(record, message)
+      self.transform_errors <<
+        TransformError.new(:model_class => record.class.name, :message => message)
+    end
+  end
+
+  class TransformError
+    include ::DataMapper::Resource
+
+    storage_names[:default] = storage_names[:mdes_warehouse_working] =
+      storage_names[:mdes_warehouse_reporting] = 'wh_transform_error'
+
+    property :id,          Serial
+    property :message,     Text,   :required => true
+    property :model_class, String, :length => 255
+
+    belongs_to :transform_status, TransformStatus, :required => true
+  end
+
+  TransformError.finalize
+  TransformStatus.finalize
+end

data/lib/ncs_navigator/warehouse/transformers.rb

@@ -13,7 +13,6 @@ module NcsNavigator::Warehouse
   module Transformers
     autoload :Database,        'ncs_navigator/warehouse/transformers/database'
     autoload :EnumTransformer, 'ncs_navigator/warehouse/transformers/enum_transformer'
-    autoload :TransformStatus, 'ncs_navigator/warehouse/transformers/transform_status'
     autoload :VdrXml,          'ncs_navigator/warehouse/transformers/vdr_xml'
   end
 end

data/lib/ncs_navigator/warehouse/transformers/database.rb

@@ -125,7 +125,7 @@ module NcsNavigator::Warehouse::Transformers
         log.debug("Executing query for producer #{rp.name}:\n#{rp.query}")
         repository.adapter.select(rp.query).each do |row|
           row_count += 1
-          [*rp.row_processor.call(row)].each do |result|
+          [*rp.row_processor.call(row)].compact.each do |result|
             yield result
             result_count += 1
             shell.back_up_and_say(24, "(%-6d in / %-6d out)" % [row_count, result_count])
@@ -198,7 +198,7 @@ module NcsNavigator::Warehouse::Transformers
         if columns.empty?
           @ignored_columns ||= []
         else
-          @ignored_columns = columns.collect(&:to_sym)
+          @ignored_columns = columns.collect(&:to_s)
         end
       end
 
@@ -242,116 +242,130 @@ module NcsNavigator::Warehouse::Transformers
       end
 
       ##
-      # Performs automatic conversion from a row struct to a
-      # an instance of a particular warehouse model. This conversion
-      # uses several heuristics to apply values from the row to the
-      # model instance. Each column in the row will be converted into
-      # at most one model property value. In the order that they are
-      # applied, the heuristics are:
+      # Define a translation from the results of a query into exactly
+      # one warehouse record per result row. This method, while more
+      # restrictive than {#produce_records}, allows for rapidly
+      # mapping data which is already a close match for MDES records.
       #
+      # The mapping uses a series of heuristics to map from the
+      # columns in each query result row to at most one MDES variable
+      # from the specified model.
+      #
+      #   * If the column appears as a key in the `:column_map` hash,
+      #     use the associated value as the target variable name.
       #   * If there's a `:prefix` option, the column is named {X},
-      #     and there's a property named {prefix}{X}, use that
-      #     property.
-      #   * If the column is named {X} and there's a property named
-      #     {X}, use that property.
-      #   * If the column is named {X}_code and there's a property
-      #     named {X}, use that property.
-      #   * If the column is named {X}_code and there's a property
-      #     named {X}_id, use that property.
+      #     and there's a variable named {prefix}{X}, use that
+      #     variable.
+      #   * If the column is named {X} and there's a variable named
+      #     {X}, use that variable.
+      #   * If the column is named {X}_code and there's a variable
+      #     named {X}, use that variable.
+      #   * If the column is named {X}_code and there's a variable
+      #     named {X}_id, use that variable.
       #   * If the column is named {X}_other and there's a property
-      #     named {X}_oth, use that property.
-      #
-      # Separately, any property value in the instance may be
-      # explicitly set via a hash passed as the `:explicit`
-      # option. Property values in `:explicit` take precedence over
-      # the heuristically-determined values.
+      #     named {X}_oth, use that variable.
       #
-      # @param [Class] model the warehouse model class (e.g.,
-      #   `NcsNavigator::Warehouse::Models::TwoPointZero::Person`)
-      # @param [Object] row a DataMapper row struct that is the source
-      #   of the data for the instance. (This is the kind of object
-      #   that is yielded to {#produce_records} blocks.)
-      # @param [Hash] options Options controlling the created
-      #   instance.
+      # @param [Symbol] name the name of this producer; if you don't
+      #   specify a `:query`, the default is to return every row from
+      #   the application table with this name.
+      # @param [Class] model the warehouse model to which results of
+      #   the query will be mapped.
+      # @param [Hash] options
+      # @option options :query [String] the query to execute for this
+      #   producer. If not specified, the query is `"SELECT * FROM #{name}"`.
       #
       # @option options :prefix [String] a prefix to use when looking
       #   for matching property values. (See above.)
       # @option options :column_map [Hash<Symbol, Symbol>] explicit
-      #   mapping from column name to model property name. This
-      #   mapping is consulted before the heuristics are applied and
-      #   before `:property_values` is used.
-      # @option options :property_values [Hash<Symbol, Object>]
-      #   explicit values to use. Keys are model property names and
-      #   values are the desired values. Any values in this hash trump
-      #   the heuristically-determined values.
+      #   mapping from column name to MDES variable name. This
+      #   mapping is consulted before the heuristics are applied.
       # @option options :on_unused [:ignore,:fail] what to do when
       #   there are columns in the row which are not used.
-      # @option options :ignored_columns [Array<String,Symbol>]
-      #   columns to consider "used" even if the heuristic doesn't
-      #   match them to anything.
+      # @option options :ignored_columns [Array<[String,Symbol]>]
+      #   columns to consider "used" even if the heuristic or the
+      #   column map don't match them to anything.
       #
-      # @return [Object] an instance of `model`.
-      def model_row(model, row, options={})
-        unused_behavior = options[:on_unused] || on_unused_columns
-        pv, unused = create_property_values(model, row, options)
+      # @return [void]
+      def produce_one_for_one(name, model, options={})
+        options[:on_unused] ||= on_unused_columns
+        options[:column_map] =
+          (options[:column_map] || {}).inject({}) { |h, (k, v)| h[k.to_s] = v.to_s; h }
+        options[:ignored_columns] =
+          (options[:ignored_columns] || []).collect(&:to_s) + ignored_columns
+
+        record_producers << OneForOneProducer.new(name, options.delete(:query), model, options)
+      end
+    end
 
-        unused -= ignored_columns
-        unused -= options[:ignored_or_used].collect(&:to_sym) if options[:ignored_or_used]
+    ##
+    # The class ecapsulating one call to {DSL#produce_records}
+    class RecordProducer < Struct.new(:name, :query, :row_processor)
+      def query
+        super || "SELECT * FROM #{name}"
+      end
+    end
 
-        if unused_behavior == :fail && !unused.empty?
-          raise UnusedColumnsForModelError.new(unused)
-        end
-        model.new(pv)
+    ##
+    # The class encapsulating one call to {DSL#produce_one_for_one}
+    class OneForOneProducer < RecordProducer
+      attr_reader :model, :options
+
+      def initialize(name, query, model, options)
+        super(name, query, self)
+        @model = model
+        @options = options
       end
 
       ##
-      # Returns a two-member array. The first member is the matched
-      # property values. The second is the columns from the row which
-      # were not matched to anything.
-      def create_property_values(model, row, options)
-        column_map = (options[:column_map] || {}).inject({}) { |h, (k, v)| h[k.to_s] = v.to_s; h }
+      # Produces a single instance of {#model} using the values in the
+      # row as mapped by {#column_map}.
+      def convert_row(row)
+        col_map = column_map(row.members)
+        unused = row.members.collect(&:to_s) - col_map.keys - options[:ignored_columns]
 
-        pv = (options[:property_values] || {}).inject({}) { |h, (k, v)| h[k.to_s] = v; h }
-        column_map.values.each { |prop| pv.delete(prop) }
+        if options[:on_unused] == :fail && !unused.empty?
+          raise UnusedColumnsForModelError.new(unused)
+        end
+        model.new(
+          col_map.inject({}) { |pv, (col_name, var_name)| pv[var_name] = row[col_name]; pv }
+        )
+      end
+      alias :call :convert_row
 
+      ##
+      # @param [Array<String>] column_names
+      # @return [Hash<String, String>] a mapping from the given
+      #   column names to MDES variable names for the configured
+      #   model. This mapping reflects both the configured explicit
+      #   mapping and the heuristic.
+      def column_map(column_names)
         available_props = model.properties.collect { |p| p.name.to_s }
-        available_props -= pv.keys
+        available_props -= options[:column_map].values
 
-        unused = []
-        row.members.each do |column|
+        column_names.inject(options[:column_map].dup) do |map, column|
+          column = column.to_s
           prop =
-            if column_map[column.to_s]
-              column_map[column.to_s]
-            else
+            unless options[:column_map][column]
               [
                 [//,        ''],
                 [/_code$/,  ''],
                 [/_code$/,  '_id'],
                 [/_other$/, '_oth'],
-              ].collect do |pattern, substution|
+              ].collect do |pattern, substitution|
                 if column =~ pattern
                   prefixed_property_name(available_props,
-                    column.to_s.sub(pattern, substution), options[:prefix])
+                    column.sub(pattern, substitution), options[:prefix])
                 end
               end.compact.first
             end
           if prop
             available_props.delete(prop)
-            pv[prop] = row[column]
-          else
-            unused << column.to_sym
+            map[column] = prop
           end
+          map
         end
-        [pv, unused]
       end
-      private :create_property_values
 
-      ##
-      # Determines if the model has a property with the given name,
-      # with or without the prefix.
-      #
-      # @return [String,nil] the name of an existing property on the
-      #   model, either with or without the prefix; or nil.
       def prefixed_property_name(available_props, name, prefix)
         if prefix && available_props.include?(prop = "#{prefix}#{name}")
           prop
@@ -362,14 +376,6 @@ module NcsNavigator::Warehouse::Transformers
       private :prefixed_property_name
     end
 
-    ##
-    # @private
-    class RecordProducer < Struct.new(:name, :query, :row_processor)
-      def query
-        super || "SELECT * FROM #{name}"
-      end
-    end
-
     ##
     # Methods to assist with using classes that mix in {Database}.
     module Factory
@@ -378,7 +384,7 @@ module NcsNavigator::Warehouse::Transformers
       #   enumerable.
       # @see EnumTransformer
       def create_transformer(configuration, options={})
-        EnumTransformer.new(new(configuration, options))
+        EnumTransformer.new(configuration, new(configuration, options))
       end
     end
 
@@ -388,7 +394,7 @@ module NcsNavigator::Warehouse::Transformers
       def initialize(unused)
         super(
           "#{unused.size} unused column#{'s' unless unused.size == 1} when building model. " +
-          "Use :ignored_or_used => %w(#{unused.join(' ')}) or :unused => :ignore to suppress this error.")
+          "Use :ignored_columns => %w(#{unused.join(' ')}) or :on_unused => :ignore to suppress this error.")
         @unused = unused
       end
     end

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -1,5 +1,7 @@
 require 'ncs_navigator/warehouse'
 
+require 'forwardable'
+
 module NcsNavigator::Warehouse::Transformers
   ##
   # A transformer that accepts a series of model instances in the form
@@ -7,16 +9,26 @@ module NcsNavigator::Warehouse::Transformers
   # array, or it might be a custom class that streams through
   # thousands of instances without having them all in memory at once.
   class EnumTransformer
+    extend Forwardable
+
     ##
     # @return [Enumerable] the enumeration that will be transformed.
     attr_reader :enum
 
+    def_delegators :@configuration, :log, :shell
+
     ##
+    # @param [Configuration] configuration
     # @param [Enumerable] enum
-    def initialize(enum)
+    def initialize(configuration, enum)
+      @configuration = configuration
       @enum = enum
     end
 
+    def name
+      "EnumTransformer for #{enum.class}"
+    end
+
     ##
     # Takes each in-memory record provided by the configured
     # `Enumerable`, validates it, and saves it if it is valid.
@@ -26,14 +38,17 @@ module NcsNavigator::Warehouse::Transformers
     def transform(status)
       enum.each do |record|
         if record.valid?
+          log.debug("Saving valid record #{record_ident record}.")
           begin
             unless record.save
-              status.unsuccessful_record(
-                record, "Could not save. #{record_ident(record)}.")
+              msg = "Could not save. #{record_ident(record)}."
+              log.error msg
+              status.unsuccessful_record(record, msg)
             end
           rescue => e
-            status.unsuccessful_record(
-              record, "Error on save. #{e.class}: #{e}. #{record_ident(record)}.")
+            msg = "Error on save. #{e.class}: #{e}. #{record_ident(record)}."
+            log.error msg
+            status.unsuccessful_record(record, msg)
           end
         else
           messages = record.errors.keys.collect { |prop|
@@ -42,9 +57,11 @@ module NcsNavigator::Warehouse::Transformers
               "#{e} (#{prop}=#{v.inspect})."
             }
           }.flatten
-          status.unsuccessful_record(
-            record, "Invalid record. #{messages.join(' ')} #{record_ident(record)}.")
+          msg = "Invalid record. #{messages.join(' ')} #{record_ident(record)}."
+          log.error msg
+          status.unsuccessful_record(record, msg)
         end
+        status.record_count += 1
       end
     end
 
@@ -52,7 +69,8 @@ module NcsNavigator::Warehouse::Transformers
 
     def record_ident(rec)
       # No composite keys in the MDES
-      '%s %s=%s' % [rec.class.name.demodulize, rec.class.key.first.name, rec.key.first.inspect]
+      '%s %s=%s' % [
+        rec.class.name.demodulize, rec.class.key.first.name, rec.key.try(:first).inspect]
     end
   end
 end