ncs_mdes_warehouse 0.0.1 → 0.0.2

data/.gitignore

@@ -5,3 +5,4 @@ docs
 reports
 *.sqlite3
 logs
+pkg

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.0.2
+-----
+
+- Added `model_row` helper in Transformers::Database for
+  heuristic-based semi-automatic mapping of transactional data to
+  warehouse entries.
+
 0.0.1
 -----
 

data/lib/ncs_navigator/warehouse/transformers/database.rb

@@ -2,6 +2,7 @@ require 'ncs_navigator/warehouse'
 
 require 'active_support/core_ext/string'
 require 'forwardable'
+require 'data_mapper'
 
 module NcsNavigator::Warehouse::Transformers
   ##
@@ -93,6 +94,7 @@ module NcsNavigator::Warehouse::Transformers
     def repository
       @repository ||=
         begin
+          log.debug("Connecting DM repo #{repository_name.inspect} with #{connection_parameters.merge('password' => 'SUPPRESSED').inspect}")
           ::DataMapper.setup(repository_name, connection_parameters)
           ::DataMapper.repository(repository_name)
         end
@@ -173,6 +175,33 @@ module NcsNavigator::Warehouse::Transformers
         @repository_name ||= name.demodulize.underscore.to_sym
       end
 
+      ##
+      # What to do if are encountered by {#model_row}. Default is
+      # `:ignore` but may be set to `:fail` for all producers using
+      # this method. The value set here may be overridden per
+      # {#model_row} call.
+      #
+      # @return [:ignore, :fail]
+      def on_unused_columns(setting=nil)
+        if setting
+          @on_unused_columns = setting
+        else
+          @on_unused_columns ||= :ignore
+        end
+      end
+
+      ##
+      # Indicates a set of columns which which should not be checked
+      # for used-ness by {#model_row}, even if the `:unused` option is
+      # set to `:fail`.
+      def ignored_columns(*columns)
+        if columns.empty?
+          @ignored_columns ||= []
+        else
+          @ignored_columns = columns.collect(&:to_sym)
+        end
+      end
+
       ##
       # Defines the bcdatabase specification to use when connecting to
       # the database for this enumeration.
@@ -211,6 +240,126 @@ module NcsNavigator::Warehouse::Transformers
       def produce_records(name, options={}, &logic)
         record_producers << RecordProducer.new(name, options[:query], logic)
       end
+
+      ##
+      # Performs automatic conversion from a row struct to a
+      # an instance of a particular warehouse model. This conversion
+      # uses several heuristics to apply values from the row to the
+      # model instance. Each column in the row will be converted into
+      # at most one model property value. In the order that they are
+      # applied, the heuristics are:
+      #
+      #   * If there's a `:prefix` option, the column is named {X},
+      #     and there's a property named {prefix}{X}, use that
+      #     property.
+      #   * If the column is named {X} and there's a property named
+      #     {X}, use that property.
+      #   * If the column is named {X}_code and there's a property
+      #     named {X}, use that property.
+      #   * If the column is named {X}_code and there's a property
+      #     named {X}_id, use that property.
+      #   * If the column is named {X}_other and there's a property
+      #     named {X}_oth, use that property.
+      #
+      # Separately, any property value in the instance may be
+      # explicitly set via a hash passed as the `:explicit`
+      # option. Property values in `:explicit` take precedence over
+      # the heuristically-determined values.
+      #
+      # @param [Class] model the warehouse model class (e.g.,
+      #   `NcsNavigator::Warehouse::Models::TwoPointZero::Person`)
+      # @param [Object] row a DataMapper row struct that is the source
+      #   of the data for the instance. (This is the kind of object
+      #   that is yielded to {#produce_records} blocks.)
+      # @param [Hash] options Options controlling the created
+      #   instance.
+      #
+      # @option options :prefix [String] a prefix to use when looking
+      #   for matching property values. (See above.)
+      # @option options :column_map [Hash<Symbol, Symbol>] explicit
+      #   mapping from column name to model property name. This
+      #   mapping is consulted before the heuristics are applied and
+      #   before `:property_values` is used.
+      # @option options :property_values [Hash<Symbol, Object>]
+      #   explicit values to use. Keys are model property names and
+      #   values are the desired values. Any values in this hash trump
+      #   the heuristically-determined values.
+      # @option options :on_unused [:ignore,:fail] what to do when
+      #   there are columns in the row which are not used.
+      # @option options :ignored_columns [Array<String,Symbol>]
+      #   columns to consider "used" even if the heuristic doesn't
+      #   match them to anything.
+      #
+      # @return [Object] an instance of `model`.
+      def model_row(model, row, options={})
+        unused_behavior = options[:on_unused] || on_unused_columns
+        pv, unused = create_property_values(model, row, options)
+
+        unused -= ignored_columns
+        unused -= options[:ignored_or_used].collect(&:to_sym) if options[:ignored_or_used]
+
+        if unused_behavior == :fail && !unused.empty?
+          raise UnusedColumnsForModelError.new(unused)
+        end
+        model.new(pv)
+      end
+
+      ##
+      # Returns a two-member array. The first member is the matched
+      # property values. The second is the columns from the row which
+      # were not matched to anything.
+      def create_property_values(model, row, options)
+        column_map = (options[:column_map] || {}).inject({}) { |h, (k, v)| h[k.to_s] = v.to_s; h }
+
+        pv = (options[:property_values] || {}).inject({}) { |h, (k, v)| h[k.to_s] = v; h }
+        column_map.values.each { |prop| pv.delete(prop) }
+
+        available_props = model.properties.collect { |p| p.name.to_s }
+        available_props -= pv.keys
+
+        unused = []
+        row.members.each do |column|
+          prop =
+            if column_map[column.to_s]
+              column_map[column.to_s]
+            else
+              [
+                [//,        ''],
+                [/_code$/,  ''],
+                [/_code$/,  '_id'],
+                [/_other$/, '_oth'],
+              ].collect do |pattern, substution|
+                if column =~ pattern
+                  prefixed_property_name(available_props,
+                    column.to_s.sub(pattern, substution), options[:prefix])
+                end
+              end.compact.first
+            end
+          if prop
+            available_props.delete(prop)
+            pv[prop] = row[column]
+          else
+            unused << column.to_sym
+          end
+        end
+        [pv, unused]
+      end
+      private :create_property_values
+
+      ##
+      # Determines if the model has a property with the given name,
+      # with or without the prefix.
+      #
+      # @return [String,nil] the name of an existing property on the
+      #   model, either with or without the prefix; or nil.
+      def prefixed_property_name(available_props, name, prefix)
+        if prefix && available_props.include?(prop = "#{prefix}#{name}")
+          prop
+        elsif available_props.include?(name)
+          name
+        end
+      end
+      private :prefixed_property_name
     end
 
     ##
@@ -232,5 +381,16 @@ module NcsNavigator::Warehouse::Transformers
         EnumTransformer.new(new(configuration, options))
       end
     end
+
+    class UnusedColumnsForModelError < StandardError
+      attr_accessor :unused
+
+      def initialize(unused)
+        super(
+          "#{unused.size} unused column#{'s' unless unused.size == 1} when building model. " +
+          "Use :ignored_or_used => %w(#{unused.join(' ')}) or :unused => :ignore to suppress this error.")
+        @unused = unused
+      end
+    end
   end
 end

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.0.1'
+    VERSION = '0.0.2'
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'ncs_mdes', '~> 0.4', '>= 0.4.2'
   s.add_dependency 'ncs_navigator_configuration', '~> 0.1'
 
-  s.add_dependency 'activesupport', '~> 3.0.9'
+  s.add_dependency 'activesupport', '~> 3.0'
   s.add_dependency 'i18n', '~> 0.6' # required by activesupport
 
   s.add_dependency 'thor', '~> 0.14.6'

data/spec/ncs_navigator/warehouse/transformers/database_spec.rb

@@ -167,6 +167,15 @@ module NcsNavigator::Warehouse::Transformers
 
           acc.should == %w(A)
         end
+
+        it 'can execute just one producer by calling :to_a' do
+          execute_sql(
+            "INSERT INTO people (id) VALUES ('B')",
+            "INSERT INTO more_people (id) VALUES ('Q')"
+          )
+
+          enumerator.to_a(:people).should == %w(B)
+        end
       end
     end
 
@@ -190,5 +199,181 @@ module NcsNavigator::Warehouse::Transformers
         end
       end
     end
+
+    describe Database::DSL do
+      describe '#on_unused_columns' do
+        it 'defaults to :ignore' do
+          sample_class.on_unused_columns.should == :ignore
+        end
+
+        it 'can be set' do
+          sample_class { on_unused_columns :fail }.on_unused_columns.should == :fail
+        end
+      end
+
+      describe '#model_row' do
+        # DataMapper models can't be anonymous
+        module Database::DSL::TestModels
+          class Address
+            include ::DataMapper::Resource
+
+            property   :address_id, String, :key => true
+            property   :street, NcsNavigator::Warehouse::DataMapper::NcsString
+            property   :address_type,
+                       NcsNavigator::Warehouse::DataMapper::NcsString,
+                       { :required => true, :length => 1..2, :set => ["1", "2", "3", "4", "5", "-5", "-6", "-4"] }
+            property   :address_type_oth,
+                       NcsNavigator::Warehouse::DataMapper::NcsString,
+                       { :pii => :possible, :length => 0..255 }
+          end
+
+          Address.finalize
+        end
+
+        let(:address_model) { Database::DSL::TestModels::Address }
+
+        let(:options) { {} }
+        let(:cls) { sample_class }
+
+        def make_row(contents)
+          Struct.new(*contents.keys).new.tap do |r|
+            contents.each_pair do |c, v|
+              r.send("#{c}=", v)
+            end
+          end
+        end
+
+        def model_row(row)
+          cls.model_row(address_model, make_row(row), options)
+        end
+
+        it 'maps a column to the property with the same name' do
+          model_row(:street => '123 Anymain Dr.').street.should == '123 Anymain Dr.'
+        end
+
+        it 'maps a column named {X}_code to a property named {X}' do
+          model_row(:address_type_code => '5').address_type.should == '5'
+        end
+
+        it 'maps a column named {X}_code to a property named {X}_id' do
+          model_row(:address_code => '-7').address_id.should == '-7'
+        end
+
+        it 'maps a column named {X}_other to a property named {X}_oth' do
+          model_row(:address_type_other => 'Elephant graveyard').address_type_oth.
+            should == 'Elephant graveyard'
+        end
+
+        describe 'and unused columns' do
+          it 'fails with unused columns if requested' do
+            options[:on_unused] = :fail
+            begin
+              model_row(:address_type => '-5', :address_length => '6')
+              fail "Exception not thrown"
+            rescue Database::UnusedColumnsForModelError => e
+              e.unused.should == [:address_length]
+            end
+          end
+
+          it 'does not fail if the unused column is ignored in the options' do
+            options[:on_unused] = :fail
+            options[:ignored_or_used] = %w(address_length)
+            lambda { model_row(:address_type => '-5', :address_length => '6') }.
+              should_not raise_error
+          end
+
+          it 'does not fail if the unused column is ignored at the class level' do
+            options[:on_unused] = :fail
+            cls.ignored_columns 'address_length'
+            lambda { model_row(:address_type => '-5', :address_length => '6') }.
+              should_not raise_error
+          end
+
+          describe 'when #on_unused_columns is set to :fail' do
+            let(:cls) {
+              sample_class do
+                on_unused_columns :fail
+              end
+            }
+
+            it 'fails appropriately' do
+              begin
+                model_row(:address_type => '-5', :address_length => '6')
+                fail "Exception not thrown"
+              rescue Database::UnusedColumnsForModelError => e
+                e.unused.should == [:address_length]
+              end
+            end
+
+            it 'does not fail if the global setting is overridden' do
+              options[:on_unused] = :ignore
+              lambda { model_row(:address_type => '-5', :address_length => '6') }.
+                should_not raise_error
+            end
+          end
+        end
+
+        describe 'with a prefix' do
+          before do
+            options[:prefix] = 'address_'
+          end
+
+          it 'maps a column to {prefix}_{property_name}' do
+            model_row(:type => '-4').address_type.should == '-4'
+          end
+
+          it 'still maps a column to {property_name} if there is no prefixed version' do
+            model_row(:street => '123 Anymain Dr.').street.should == '123 Anymain Dr.'
+          end
+
+          it 'maps a column named {X}_code to {prefix}{X}' do
+            model_row(:type_code => '1').address_type.should == '1'
+          end
+        end
+
+        describe 'with a column map' do
+          before do
+            options[:column_map] = { :street_loc => :street }
+          end
+
+          it 'uses the value specified by the map' do
+            model_row(:street_loc => '123 Anymain Dr.').street.should == '123 Anymain Dr.'
+          end
+
+          it 'prefers the mapped value to a heuristically-determined value' do
+            model_row(
+              :street_loc => '123 Anymain Dr.', :street => '456 Anywhere St.'
+            ).street.should == '123 Anymain Dr.'
+          end
+
+          it 'prefers the mapped column value to an explicit property value' do
+            options[:property_values] = { :street => '456 Anywhere St.' }
+            model_row(:street_loc => '123 Anymain Dr.').street.should == '123 Anymain Dr.'
+          end
+        end
+
+        describe 'with property value mappings' do
+          before do
+            options[:property_values] = {
+              :street => '456 Anywhere St.'
+            }
+          end
+
+          it 'prefers the explicit mapping to a column' do
+            model_row(:street => '123 Anymain Dr.').street.should == '456 Anywhere St.'
+          end
+
+          it 'reports the column as unused' do
+            options[:on_unused] = :fail
+            begin
+              model_row(:street => '123 Anymain Dr.').street.should == '456 Anywhere St.'
+              fail "Exception not thrown"
+            rescue Database::UnusedColumnsForModelError => e
+              e.unused.should include(:street)
+            end
+          end
+        end
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ncs_mdes_warehouse
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-28 00:00:00.000000000Z
+date: 2011-11-04 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ncs_mdes
-  requirement: &2154987140 !ruby/object:Gem::Requirement
+  requirement: &2159237800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -24,10 +24,10 @@ dependencies:
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *2154987140
+  version_requirements: *2159237800
 - !ruby/object:Gem::Dependency
   name: ncs_navigator_configuration
-  requirement: &2154986380 !ruby/object:Gem::Requirement
+  requirement: &2159236980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -35,21 +35,21 @@ dependencies:
         version: '0.1'
   type: :runtime
   prerelease: false
-  version_requirements: *2154986380
+  version_requirements: *2159236980
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2154985920 !ruby/object:Gem::Requirement
+  requirement: &2159236440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.0.9
+        version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2154985920
+  version_requirements: *2159236440
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &2154985420 !ruby/object:Gem::Requirement
+  requirement: &2159235940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -57,10 +57,10 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *2154985420
+  version_requirements: *2159235940
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &2154984960 !ruby/object:Gem::Requirement
+  requirement: &2159235400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -68,10 +68,10 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *2154984960
+  version_requirements: *2159235400
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &2154984420 !ruby/object:Gem::Requirement
+  requirement: &2159234800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -79,10 +79,10 @@ dependencies:
         version: 0.9.4
   type: :runtime
   prerelease: false
-  version_requirements: *2154984420
+  version_requirements: *2159234800
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2154983960 !ruby/object:Gem::Requirement
+  requirement: &2159234100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -90,10 +90,10 @@ dependencies:
         version: 1.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *2154983960
+  version_requirements: *2159234100
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &2154983480 !ruby/object:Gem::Requirement
+  requirement: &2159233480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -101,10 +101,10 @@ dependencies:
         version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2154983480
+  version_requirements: *2159233480
 - !ruby/object:Gem::Dependency
   name: data_mapper
-  requirement: &2154982940 !ruby/object:Gem::Requirement
+  requirement: &2159232820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -112,10 +112,10 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2154982940
+  version_requirements: *2159232820
 - !ruby/object:Gem::Dependency
   name: bcdatabase
-  requirement: &2154982440 !ruby/object:Gem::Requirement
+  requirement: &2159232220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -123,10 +123,10 @@ dependencies:
         version: '1.1'
   type: :runtime
   prerelease: false
-  version_requirements: *2154982440
+  version_requirements: *2159232220
 - !ruby/object:Gem::Dependency
   name: dm-postgres-adapter
-  requirement: &2154981960 !ruby/object:Gem::Requirement
+  requirement: &2159231600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -134,10 +134,10 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2154981960
+  version_requirements: *2159231600
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2154981460 !ruby/object:Gem::Requirement
+  requirement: &2159230940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -145,10 +145,10 @@ dependencies:
         version: '2.6'
   type: :development
   prerelease: false
-  version_requirements: *2154981460
+  version_requirements: *2159230940
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2154981000 !ruby/object:Gem::Requirement
+  requirement: &2159230280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -156,10 +156,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2154981000
+  version_requirements: *2159230280
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &2154953340 !ruby/object:Gem::Requirement
+  requirement: &2159225600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -167,10 +167,10 @@ dependencies:
         version: 0.7.2
   type: :development
   prerelease: false
-  version_requirements: *2154953340
+  version_requirements: *2159225600
 - !ruby/object:Gem::Dependency
   name: ci_reporter
-  requirement: &2154952780 !ruby/object:Gem::Requirement
+  requirement: &2159225100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -178,10 +178,10 @@ dependencies:
         version: 1.6.5
   type: :development
   prerelease: false
-  version_requirements: *2154952780
+  version_requirements: *2159225100
 - !ruby/object:Gem::Dependency
   name: fakefs
-  requirement: &2154952200 !ruby/object:Gem::Requirement
+  requirement: &2159224500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -189,7 +189,7 @@ dependencies:
         version: 0.4.0
   type: :development
   prerelease: false
-  version_requirements: *2154952200
+  version_requirements: *2159224500
 description: 
 email:
 - r-sutphin@northwestern.edu
@@ -529,7 +529,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1308600136922814816
+      hash: 3545445795691480899
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -538,7 +538,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1308600136922814816
+      hash: 3545445795691480899
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10