caruby-core 1.5.3 → 1.5.4

data/History.md

@@ -1,6 +1,10 @@
 This history lists major release themes. See the GitHub Commits (https://github.com/caruby/core)
 for change details.
 
+1.5.4 / 2011-07-19
+------------------
+* Add migration value filter option.
+
 1.5.3 / 2011-07-08
 ------------------
 * Clean up documentation.

data/lib/caruby/cli/command.rb

@@ -32,11 +32,12 @@ module CaRuby
       #
       # Built-in options include the following:
       # * --help : print the help message and exit
-      # * --version : print the version and exit
+      # * --verbose : print additional information to the console
       # * --log FILE : log file
       # * --debug : print debug messages to the log
       # * --file FILE: configuration file containing other options
       # * --quiet: suppress printing messages to stdout
+      #
       # This class processes these built-in options, with the exception of +--version+,
       # which is a subclass responsibility. Subclasses are responsible for
       # processing any remaining options.

data/lib/caruby/database/reader.rb

@@ -321,15 +321,14 @@ module CaRuby
           return
         end
         @transients << obj
-
         logger.debug { "Fetching #{obj.qp} from the database..." }
         fetched = fetch_object(obj) || return
-        # fetch_object can return obj; if so, then done
+        # fetch_object can return obj; if so, then done, otherwise, merge fetched.
         return obj if obj.equal?(fetched)
         
         logger.debug { "Fetch #{obj.qp} matched database object #{fetched}." }
         @transients.delete(obj)
-        #   recursively copy the nondomain attributes, esp. the identifer, of the fetched domain object references
+        # recursively copy the nondomain attributes, esp. the identifer, of the fetched domain object references
         merge_fetched(fetched, obj)
         # Inject the lazy loader for loadable domain reference attributes.
         persistify(obj, fetched)

data/lib/caruby/database/search_template_builder.rb

@@ -24,14 +24,14 @@ module CaRuby
       logger.debug { "Building search template for #{obj.qp}..." }
       hash ||= obj.value_hash(obj.class.searchable_attributes)
       # the searchable attribute => value hash
-      ref_hash, nonref_hash = hash.hash_partition { |attr, value| Resource === value }
+      rh, nrh = hash.split { |attr, value| Resource === value }
       # make the search template from the non-reference attributes
-      tmpl = obj.class.new.merge_attributes(nonref_hash)
+      tmpl = obj.class.new.merge_attributes(nrh)
       # get references for the search template
-      unless ref_hash.empty? then
-        logger.debug { "Collecting search reference parameters for #{obj.qp} from attributes #{ref_hash.keys.to_series}..." }
+      unless rh.empty? then
+        logger.debug { "Collecting search reference parameters for #{obj.qp} from attributes #{rh.keys.to_series}..." }
       end
-      ref_hash.each { |attr, ref| add_search_template_reference(tmpl, ref, attr) }
+      rh.each { |attr, ref| add_search_template_reference(tmpl, ref, attr) }
       tmpl
     end
 

data/lib/caruby/database/sql_executor.rb

@@ -34,6 +34,9 @@ module CaRuby
     # @option opts [String] :database_driver_class the optional DBI connect driver class name
     # @raise [CaRuby::ConfigurationError] if an option is invalid
     def initialize(opts)
+      if opts.empty? then
+        raise CaRuby::ConfigurationError.new("The caRuby database connection properties were not found.") 
+      end
       app_host = Options.get(:host, opts, 'localhost')
       db_host = Options.get(:database_host, opts, app_host)
       db_type = Options.get(:database_type, opts, 'mysql')
@@ -52,9 +55,10 @@ module CaRuby
         :database_type => db_type,
         :database_port => db_port,
         :database_driver => db_driver,
-        :database_driver_class => @driver_class
-      }      
-      logger.debug { "Database connection options (excluding password): #{eff_opts.qp}" }
+        :database_driver_class => @driver_class,
+        :address => @address
+      }
+      logger.debug { "Database connection parameters (excluding password): #{eff_opts.qp}" }
     end
 
     # Connects to the database, yields the DBI handle to the given block and disconnects.

data/lib/caruby/database/writer.rb

@@ -139,6 +139,7 @@ module CaRuby
       def ensure_exists(obj)
         raise ArgumentError.new("Database ensure_exists is missing a domain object argument") if obj.nil_or_empty?
         obj.enumerate { |ref| find(ref, :create) unless ref.identifier }
+        
       end
 
       private
@@ -286,7 +287,6 @@ module CaRuby
           logger.debug { "Ensuring that created #{obj.qp} references exist: #{refs.qp}..." } unless refs.empty?
           refs.each { |ref| ensure_exists(ref) }
         end
-        
         obj
       end
       
@@ -350,7 +350,7 @@ module CaRuby
       # * the attribute references a {#pending_create?} save context.
       #
       # @param obj (see #create)
-      # @param [Attribute] attr_md candidate attribute metadata
+      # @param [Attribute] attr_md the candidate attribute metadata
       # @return [Boolean] whether the attribute should not be included in the create template
       def exclude_pending_create_attribute?(obj, attr_md)
         attr_md.independent? and
@@ -413,7 +413,7 @@ module CaRuby
         # update a cascaded dependent by updating the owner
         owner = cascaded_owner(obj)
         if owner then return update_cascaded_dependent(owner, obj) end
-        
+
         # Not cascaded dependent; update using a template,
         tmpl = build_update_template(obj)
         # call the caCORE service with an obj update template
@@ -559,10 +559,9 @@ module CaRuby
           logger.debug { "Updating saved #{target} to store unsaved attributes..." }
           # call update_object(saved) rather than update(saved) to bypass the redundant update check
           perform(:update, target) { update_object(target) }
-        else
-          # recursively save the dependents
-          save_changed_dependents(target)
         end
+        # recursively save the dependents
+        save_changed_dependents(target)
       end
       
       # Synchronizes the given saved target result source with the database content.
@@ -685,9 +684,9 @@ module CaRuby
       # @param [Resource] obj the owner domain object
       def save_changed_dependents(obj)
         obj.class.dependent_attributes.each do |attr|
-          deps = obj.send(attr).to_enum
-          logger.debug { "Saving the #{obj} #{attr} dependents #{deps.qp(:single_line)} which have changed..." } unless deps.empty?
-          deps.each { |dep| save_dependent_if_changed(obj, attr, dep) }
+          deps = obj.send(attr)
+          logger.debug { "Saving the #{obj} #{attr} dependents #{deps.to_enum.qp(:single_line)} which have changed..." } unless deps.nil_or_empty?
+          deps.enumerate { |dep| save_dependent_if_changed(obj, attr, dep) }
         end
       end
       

data/lib/caruby/domain/attribute.rb

@@ -127,7 +127,7 @@ module CaRuby
       # The inverse relation is symmetric, i.e. the inverse of the referenced Attribute
       # is set to this Attribute's subject attribute.
       #
-      # @param attribute the inverse attribute
+      # @param [Symbol, nil] attribute the inverse attribute
       def inverse=(attribute)
         return if inverse == attribute
         # if no attribute, then the clear the existing inverse, if any
@@ -522,9 +522,12 @@ module CaRuby
       def clear_inverse
         return unless @inv_md
         logger.debug { "Clearing #{@declarer.qp}.#{self} inverse #{type.qp}.#{inverse}..." }
-        inv_inv_md = @inv_md.inverse_metadata
+        # Capture the inverse before unsetting it.
+        inv_md = @inv_md
+        # Unset the inverse.
         @inv_md = nil
-        if inv_inv_md then inv_inv_md.inverse = nil end
+        # Clear the inverse of the inverse.
+        inv_md.inverse = nil
         logger.debug { "Cleared #{@declarer.qp}.#{self} inverse." }
       end
       

data/lib/caruby/domain/attributes.rb

@@ -530,8 +530,13 @@ module CaRuby
       def remove_attribute(attribute)
         std_attr = standard_attribute(attribute)
         # if the attribute is local, then delete it, otherwise filter out the superclass attribute
-        if @local_attr_md_hash.delete(std_attr) then
+        attr_md = @local_attr_md_hash.delete(std_attr)
+        if attr_md then
+          # clear the inverse, if any
+          attr_md.inverse = nil
+          # remove from the mandatory attributes, if necessary
           @local_mndty_attrs.delete(std_attr)
+          # remove from the attribute => metadata hash
           @local_std_attr_hash.delete_if { |aliaz, attr| attr == std_attr }
         else
           # Filter the superclass hashes.

data/lib/caruby/domain/importer.rb

@@ -39,8 +39,8 @@ module CaRuby
         logger.debug { "Detecting whether #{symbol} is a #{@pkg} Java class..." }
         # Append the symbol to the package to make the Java class name.
         begin
-          klass = eval "Java::#{@pkg}.#{symbol}"
-          resource_import klass
+          klass = java_import "#{@pkg}.#{symbol}"
+          resource_import(klass)
         rescue NameError
           logger.debug { "#{symbol} is not recognized as a #{@pkg} Java class - #{$!}\n#{caller.qp}." }
           super
@@ -53,7 +53,7 @@ module CaRuby
       # The Java class is assumed to be defined in this module's package.
       # This module's mixin is added to the class.
       #
-      # @param [String] class_or_name the source directory
+      # @param [Class] klass the source directory
       # @raise [NameError] if the symbol does not correspond to a Java class
       #   in this module's package
       def resource_import(klass)

data/lib/caruby/import/java.rb

@@ -264,13 +264,13 @@ class Class
   # * If the class argument is a Java class or a Java class name, then
   #   the Ruby class is the JRuby wrapper for the Java class.
   #
-  # @param [Class, String] the class or class name
+  # @param [Class, String] class_or_name the class or class name
   # @return [Class] the corresponding Ruby class
-  def self.to_ruby(klass)
-    case klass
-      when Class then klass
-      when String then Java.module_eval(klass)
-      else to_ruby(klass.name)
+  def self.to_ruby(class_or_name)
+    case class_or_name
+      when Class then class_or_name
+      when String then eval to_ruby_name(class_or_name)
+      else to_ruby(class_or_name.name)
     end
   end
 
@@ -364,6 +364,18 @@ class Class
   private
   
   OBJ_INST_MTHDS = Object.instance_methods
+  
+  # @param [String] jname the fully-qualified Java class or interface name
+  # @return [String] the JRuby class or module name
+  # #example
+  #   Java.to_ruby_class_name('com.test.Sample') #=> Java::ComTest::Sample
+  def self.to_ruby_name(jname)
+    path = jname.split('.')
+    return "Java::#{jname}" if path.size == 1
+    cname = path[-1]
+    pkg = path[0...-1].map { |s| s.capitalize_first }.join
+    "Java::#{pkg}::#{cname}"
+  end
 end
 
 class Array

data/lib/caruby/json/deserializer.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+module CaRuby
+  module JSON
+    # JSON => {Resource} deserializer.
+    module Deserializer
+      # @param [String] json the JSON to deserialize
+      # @return [Resource] the deserialized object
+      def json_create(json)
+        # Make the new object from the json data attribute => value hash.
+        new(json['data'])
+      end
+    end
+  end
+end

data/lib/caruby/json/serializer.rb

@@ -0,0 +1,27 @@
+module CaRuby
+  module JSON
+    # {Resource} => JSON serializer.
+    module Serializer
+      # @param args the JSON serialization options
+      # @return [String] the JSON representation of this {Resource}
+      def to_json(*args)
+        database.lazy_loader.disable do
+          # The JSON class must be scoped by the domain module, not the Java package, in order
+          # to recognize the Resource JSON hooks.
+          # The data is the attribute => value hash.
+         {'json_class' => [self.class.domain_module, self.class.name.demodulize].join('::'),
+            'data' => value_hash(self.class.nonowner_attributes)
+          }.to_json(*args)
+        end
+      end
+    end
+  end
+end
+
+module Enumerable
+  # @param args the JSON serialization options
+  # @return [String] the JSON representation of this Enumerable as an array
+  def to_json(*args)
+    to_a.to_json(*args)
+  end
+end

data/lib/caruby/migration/migratable.rb

@@ -84,16 +84,13 @@ module CaRuby
     def migrate(row, migrated)
     end
 
-    # Returns whether this migration target domain object is valid. The default is true
-    # if this domain object either has no owner or its owner is valid.
+    # Returns whether this migration target domain object is valid. The default is true.
     # A migration shim should override this method on the target if there are conditions
     # which determine whether the migration should be skipped for this target object.
     #
     # @return [Boolean] whether this migration target domain object is valid
     def migration_valid?
-      # check that the owner is be valid
-      ownr = owner
-      ownr.nil? or ownr.migration_valid?
+      true
     end
 
     # Migrates this domain object's migratable references. This method is called by the
@@ -115,7 +112,7 @@ module CaRuby
     #
     # @param [{Symbol => Object}] row the input row field => value hash
     # @param [<Resource>] migrated the migrated instances, including this Resource
-    # @param [{Symbol => String}] mth_hash a hash that associates this domain object's
+    # @param [{Symbol => String}, nil] mth_hash a hash that associates this domain object's
     #   attributes to migration method names
     def migrate_references(row, migrated, mth_hash=nil)
       # migrate the owner

data/lib/caruby/migration/migrator.rb

@@ -29,10 +29,11 @@ module CaRuby
     # @param [{Symbol => Object}] opts the migration options
     # @option opts [String] :database required application {Database}
     # @option opts [String] :target required target domain class
-    # @option opts [String] :mapping required input field => caTissue attribute mapping file
-    # @option opts [String] :defaults optional caTissue attribute => value default mapping file
+    # @option opts [<String>, String] :mapping required input field => caTissue attribute mapping file(s)
+    # @option opts [<String>, String] :defaults optional caTissue attribute => value default mapping file(s)
+    # @option opts [<String>, String] :filters optional caTissue attribute input value => caTissue value filter file(s)
     # @option opts [String] :input required source file to migrate
-    # @option opts [String] :shims optional array of shim files to load
+    # @option opts [<String>, String] :shims optional shim file(s) to load
     # @option opts [String] :unique ensures that migrated objects which include the {Resource::Unique}
     # @option opts [String] :create optional flag indicating that existing target objects are ignored
     # @option opts [String] :bad optional invalid record file
@@ -74,8 +75,8 @@ module CaRuby
     alias :each :migrate
 
     private
-
-    UNIQUIFY_SHIM = File.join(File.dirname(__FILE__), 'uniquify.rb')
+    
+    REGEXP_PAT = /^\/(.*[^\\])\/([inx]+)?$/
 
     # Class {#migrate} with a {#save} block.
     def execute_save
@@ -106,10 +107,10 @@ module CaRuby
     end
 
     def parse_options(opts)
-      logger.debug { "Migrator options: #{opts.qp}" }
-      @fld_map_file = opts[:mapping]
-      raise MigrationError.new("Migrator missing required field mapping file parameter") if @fld_map_file.nil?
-      @def_file = opts[:defaults]
+      @fld_map_files = opts[:mapping]
+      raise MigrationError.new("Migrator missing required field mapping file parameter") if @fld_map_files.nil?
+      @def_files = opts[:defaults]
+      @filter_files = opts[:filters]
       @shims = opts[:shims] ||= []
       @offset = opts[:offset] ||= 0
       @input = Options.get(:input, opts)
@@ -136,14 +137,15 @@ module CaRuby
       raise MigrationError.new("No file to migrate") if @input.nil?
 
       # make a CSV loader which only converts input fields corresponding to non-String attributes
-      logger.info("Migration input file: #{@input}.")
       @loader = CsvIO.new(@input, &method(:convert))
       logger.debug { "Migration data input file #{@input} headers: #{@loader.headers.qp}" } 
 
       # create the class => path => default value hash
-      @def_hash = @def_file ? load_defaults(@def_file) : LazyHash.new { Hash.new }
+      @def_hash = @def_files ? load_defaults_files(@def_files) : {}
+      # create the class => path => default value hash
+      @filter_hash = @filter_files ? load_filter_files(@filter_files) : {}
       # create the class => path => header hash
-      fld_map = load_field_map(@fld_map_file)
+      fld_map = load_field_map_files(@fld_map_files)
       # create the class => paths hash
       @cls_paths_hash = create_class_paths_hash(fld_map, @def_hash)
       # create the path => class => header hash
@@ -208,30 +210,40 @@ module CaRuby
     
     # Adds missing klass owner classes to the migration class path hash (with empty paths).
     def add_owners(klass)
-      klass.owners.each do |owner|
-        next if @cls_paths_hash.detect_key { |other| other <= owner } or owner.abstract?
-        logger.debug { "Migrator adding #{klass.qp} owner #{owner}" }
-        @cls_paths_hash[owner] = Array::EMPTY_ARRAY
-        add_owners(owner)
+      owner = missing_owner_for(klass) || return
+      logger.debug { "Migrator adding #{klass.qp} owner #{owner}" }
+      @cls_paths_hash[owner] = Array::EMPTY_ARRAY
+      add_owners(owner)
+    end
+    
+    # @param [Class] klass the migration class
+    # @return [Class, nil] the missing class owner, if any
+    def missing_owner_for(klass)
+      # check for an owner among the current migration classes
+      return if klass.owners.any? do |owner|
+        @cls_paths_hash.detect_key { |other| other <= owner }
       end
+      # find the first non-abstract candidate owner
+      klass.owners.detect { |owner| not owner.abstract? }
     end
 
     # Creates the class => +migrate_+_<attribute>_ hash for the given klasses.
     def create_migration_method_hashes
-      # the attribute metadata => migration method hash variable
-      @attr_md_mgt_mth_map = {}
-      # the class => attribute => migration method hash variable
-      @mgt_mth_hash = {}
-      # collect the migration methods
-      customizable_class_attributes.each { |klass, attr_mds| add_migration_methods(klass, attr_mds) }
+      # the class => attribute => migration filter hash
+      @attr_flt_hash = {}
+      customizable_class_attributes.each do |klass, attr_mds|
+        flts = migration_filters(klass, attr_mds) || next
+        @attr_flt_hash[klass] = flts
+      end
+
       # print the migration shim methods
-      unless @mgt_mth_hash.empty? then
+      unless @attr_flt_hash.empty? then
         printer_hash = LazyHash.new { Array.new }
-        @mgt_mth_hash.each do |klass, attr_mth_hash|
-          mthds = attr_mth_hash.values
-          printer_hash[klass.qp] = mthds unless mthds.empty?
+        @attr_flt_hash.each do |klass, hash|
+          mths = hash.values.select { |flt| Symbol === flt }
+          printer_hash[klass.qp] = mths unless mths.empty?
         end
-        logger.info("Migration shim methods: #{printer_hash.pp_s}.")
+        logger.info("Migration shim methods: #{printer_hash.pp_s}.") unless printer_hash.empty?
       end
     end
 
@@ -275,13 +287,43 @@ module CaRuby
     # Discovers methods of the form +migrate+__attribute_ implemented for the paths
     # in the given class => paths hash the given klass. The migrate method is called
     # on the input field value corresponding to the path.
-    def add_migration_methods(klass, attr_mds)
+    def migration_filters(klass, attr_mds)
+      # the attribute => migration method hash
+      mth_hash = attribute_method_hash(klass, attr_mds)
+      proc_hash = attribute_proc_hash(klass, attr_mds)
+      return if mth_hash.empty? and proc_hash.empty?
+
+      # for each class path terminal attribute metadata, add the migration filters
+      # to the attribute metadata => filter hash
+      attr_mds.to_compact_hash do |attr_md|
+        # the filter proc
+        proc = proc_hash[attr_md.to_sym]
+        # the migration shim method
+        mth = mth_hash[attr_md.to_sym]
+        if mth then
+          if proc then
+            Proc.new do |obj, value, row|
+              # filter the value
+              fval = proc.call(value)
+              # call the migration method on the filtered value
+              obj.send(mth, fval, row) unless fval.nil?
+            end
+          else
+            # call the migration method
+            Proc.new { |obj, value, row| obj.send(mth, value, row) }
+          end
+        else
+          # call the filter
+          Proc.new { |obj, value, row| proc.call(value) }
+        end
+      end
+    end
+    
+    def attribute_method_hash(klass, attr_mds)
       # the migrate methods, excluding the Migratable migrate_references method
       mths = klass.instance_methods(true).select { |mth| mth =~ /^migrate.(?!references)/ }
-      return if mths.empty?
-
       # the attribute => migration method hash
-      attr_mth_hash = {}
+      mth_hash = {}
       mths.each do |mth|
         # the attribute suffix, e.g. name for migrate_name or Name for migrateName
         suffix = /^migrate(_)?(.*)/.match(mth).captures[1]
@@ -290,24 +332,60 @@ module CaRuby
         # the attribute for the name, or skip if no such attribute
         attr = klass.standard_attribute(attr_nm) rescue next
         # associate the attribute => method
-        attr_mth_hash[attr] = mth
+        mth_hash[attr] = mth
       end
-
-      # for each class path terminal attribute metadata, add the migration methods
-      # to the attribute metadata => migration method hash
+      mth_hash
+    end
+    
+    # @return [Attribute => {Object => Object}] the filter migration methods
+    def attribute_proc_hash(klass, attr_mds)
+      hash = @filter_hash[klass]
+      if hash.nil? then return Hash::EMPTY_HASH end
+      proc_hash = {}
       attr_mds.each do |attr_md|
-        # the attribute migration method
-        mth = attr_mth_hash[attr_md.to_sym]
-        # associate the Attribute => method
-        @attr_md_mgt_mth_map[attr_md] ||= mth if mth
+        flt = hash[attr_md.to_sym] || next
+        proc_hash[attr_md.to_sym] = to_filter_proc(flt)
+      end
+      logger.debug { "Migration filters loaded for #{klass.qp} #{proc_hash.keys.to_series}." }
+      proc_hash
+    end
+    
+    # @param [{Object => Object}] filter the config value mapping
+    # @return [Proc] the filter migration block
+    def to_filter_proc(filter)
+      # Split the filter into a straight value => value hash and a regexp => value hash.
+      ph, vh = filter.split { |k, v| k =~ REGEXP_PAT }
+      reh = {}
+      ph.each do |k, v|
+        pat, opt = REGEXP_PAT.match(k).captures
+        reopt = if opt then
+          case opt
+            when 'i' then Regexp::IGNORECASE
+            else raise MigrationError.new("Migration value filter regular expression #{k} qualifier not supported: expected 'i', found '#{opt}'")
+          end
+        end
+        re = Regexp.new(pat, reopt)
+        reh[re] = v.gsub(/\$\d/, '%s') if String === v
+      end
+      Proc.new do |value|
+        if vh.has_key?(value) then
+          vh[value]
+        else
+          # the first regex which matches the value
+          regexp = reh.detect_key { |re| value =~ re }
+          # If there is a match, then apply the filter to the match data.
+          # Otherwise, pass the value through unmodified.
+          regexp ? (reh[regexp] % $~.captures) : value
+        end
       end
-      @mgt_mth_hash[klass] = attr_mth_hash
     end
 
-    # loads the shim files.
+    # Loads the shim files.
+    #
+    # @param [<String>, String] files the file or file array
     def load_shims(files)
       logger.debug { "Loading shims with load path #{$:.pp_s}..." }
-      files.each do |file|
+      files.enumerate do |file|
         # load the file
         begin
           require file
@@ -323,7 +401,7 @@ module CaRuby
     #
     # @yield (see #migrate)
     # @yieldparam (see #migrate)
-    def migrate_rows # :yields: target
+    def migrate_rows
       # open an CSV output for bad records if the option is set
       if @bad_rec_file then
         @loader.trash = @bad_rec_file
@@ -359,7 +437,7 @@ module CaRuby
           mgt_cnt += 1
           # clear the migration state
           clear(target)
-       else
+        else
           # If there is a bad file then warn, reject and continue. Otherwise, bail.
           if @bad_rec_file then
             logger.warn("Migration not performed on record #{rec_no}.")
@@ -394,32 +472,55 @@ module CaRuby
     #
     # @param [{Symbol => Object}] row the input row field => value hash
     # @return the migrated target object if the migration is valid, nil otherwise
-    def migrate_row(row) # :yields: target
+    def migrate_row(row)
       # create an instance for each creatable class
       created = Set.new
       migrated = @creatable_classes.map { |klass| create(klass, row, created) }
       # migrate each object from the input row
       created.each { |obj| obj.migrate(row, migrated) }
-      # remove invalid migrations
-      valid, invalid = migrated.partition { |obj| migration_valid?(obj) }
-      # set the references
-      valid.each { |obj| obj.migrate_references(row, migrated, @mgt_mth_hash[obj.class]) }
+      valid = migrate_valid_references(row, migrated)
       # the target object
       target = valid.detect { |obj| @target_class === obj } || return
-      # the target is invalid if it has an invalid owner
-      return unless owner_valid?(target, valid, invalid)
       logger.debug { "Migrated target #{target}." }
       target
     end
     
+    # Sets the migration references for each valid migrated object.
+    #
+    # @param [Array] the migrated objects
+    # @return [Array] the valid migrated objects
+    def migrate_valid_references(row, migrated)
+      # Split the valid and invalid objects. The iteration is in reverse dependency order,
+      # since invalidating a dependent can invalidate the owner.
+      valid, invalid = migrated.transitive_closure(:dependents).reverse.partition do |obj|
+        if migration_valid?(obj) then
+          obj.migrate_references(row, migrated, @attr_flt_hash[obj.class])
+          true
+        else
+          obj.class.owner_attributes.each { |attr| obj.clear_attribute(attr) }
+          false
+        end
+      end
+      
+      # Go back through the valid objects in dependency order to invalidate dependents
+      # whose owner is invalid.
+      valid.reverse.each do |obj|
+        unless owner_valid?(obj, valid, invalid) then
+          invalid << valid.delete(obj)
+          logger.debug { "Invalidated migrated #{obj} since it does not have a valid owner." }
+        end
+      end
+      valid
+    end
+    
     # Returns whether the given domain object satisfies at least one of the following conditions:
-    # * it has an owner among the valid objects
     # * it does not have an owner among the invalid objects
+    # * it has an owner among the valid objects
     #
     # @param [Resource] obj the domain object to check
     # @param [<Resource>] valid the valid migrated objects
     # @param [<Resource>] invalid the invalid migrated objects
-    # @return [Boolean] whether the domain object is valid 
+    # @return [Boolean] whether the owner is valid 
     def owner_valid?(obj, valid, invalid)
       otypes = obj.class.owners
       invalid.all? { |other| not otypes.include?(other.class) } or
@@ -427,7 +528,7 @@ module CaRuby
     end
 
     # @param [Migratable] obj the migrated object
-    # @return whether the migration is successful
+    # @return [Boolean] whether the migration is successful
     def migration_valid?(obj)
       if obj.migration_valid? then
         true
@@ -477,7 +578,8 @@ module CaRuby
     # @param row (see #create)
     # @param [<Resource>] created (see #create)
     def add_defaults(obj, row, created)
-      @def_hash[obj.class].each do |path, value|
+      dh = @def_hash[obj.class] || return
+      dh.each do |path, value|
         # fill the reference path
         ref = fill_path(obj, path[0...-1], row, created)
         # set the attribute to the default value unless there is already a value
@@ -517,15 +619,19 @@ module CaRuby
       ref
     end
 
-    # Sets the obj migratable Attribute attr_md to value from the given input row.
+    # Sets the given attribute value to the filtered input value. If there is a filter
+    # defined for the attribute, then that filter is applied. If there is a migration
+    # shim method with name +migrate_+_attribute_, then than method is called on the
+    # (possibly filtered) value. The target object attribute is set to the resulting
+    # filtered value. 
+    #
+    # @param [Migratable] obj the target domain object
+    # @param [Attribute] attr_md the target attribute
+    # @param value the input value
+    # @param [{Symbol => Object}] row the input row
     def migrate_attribute(obj, attr_md, value, row)
-      # a single value can be used for both a Numeric and a String attribute; coerce the value if necessary
       # if there is a shim migrate_<attribute> method, then call it on the input value
-      mth = @attr_md_mgt_mth_map[attr_md]
-      if mth and obj.respond_to?(mth) then
-        value = obj.send(mth, value, row)
-        return if value.nil?
-      end
+      value = filter_value(obj, attr_md, value, row) || return
       # set the attribute
       begin
         obj.send(attr_md.writer, value)
@@ -534,6 +640,25 @@ module CaRuby
       end
       logger.debug { "Migrated #{obj.qp} #{attr_md} to #{value}." }
     end
+    
+    # Calls the shim migrate_<attribute> method or config filter on the input value.
+    #
+    # @param value the input value
+    # @return the input value, if there is no filter, otherwise the filtered value
+    def filter_value(obj, attr_md, value, row)
+      filter = filter_for(obj, attr_md)
+      return value if filter.nil?
+      fval = filter.call(obj, value, row)
+      unless value == fval then
+        logger.debug { "Migration filter transformed #{obj.qp} #{attr_md} value from #{value.qp} to #{fval.qp}." }
+      end
+      fval
+    end
+    
+    def filter_for(obj, attr_md)
+      flts = @attr_flt_hash[obj.class] || return
+      flts[attr_md]
+    end
 
     # @param [Resource] obj the domain object to save in the database
     # @return [Resource, nil] obj if the save is successful, nil otherwise
@@ -557,30 +682,12 @@ module CaRuby
       @rec_cnt + 1
     end
 
-    # @param [String] file the migration fields configuration file
+    # @param [<String>, String] files the migration fields mapping file or file array
     # @return [{Class => {Attribute => Symbol}}] the class => path => header hash
-    #   loaded from the configuration file
-    def load_field_map(file)
-      # load the field mapping config file
-      begin
-        config = YAML::load_file(file)
-      rescue
-        raise MigrationError.new("Could not read field map file #{file}: " + $!)
-      end
-
-      # collect the class => path => header entries
+    #   loaded from the mapping files
+    def load_field_map_files(files)
       map = LazyHash.new { Hash.new }
-      config.each do |field, attr_list|
-        next if attr_list.blank?
-        # the header accessor method for the field
-        header = @loader.accessor(field)
-        raise MigrationError.new("Field defined in migration configuration #{file} not found in input file #{@input} headers: #{field}") if header.nil?
-        # associate each attribute path in the property value with the header
-        attr_list.split(/,\s*/).each do |path_s|
-          klass, path = create_attribute_path(path_s)
-          map[klass][path] = header
-        end
-      end
+      files.enumerate { |file| load_field_map_file(file, map) }
 
       # include the target class
       map[@target_class] ||= Hash.new
@@ -602,26 +709,95 @@ module CaRuby
       map.delete_if do |klass, paths|
         klass.abstract? or classes.any? { |other| other < klass }
       end
+
       map
     end
     
-    def load_defaults(file)
+    # @param [String] file the migration fields configuration file
+    # @param [{Class => {Attribute => Symbol}}] hash the class => path => header hash
+    #   loaded from the configuration file
+    def load_field_map_file(file, hash)
       # load the field mapping config file
       begin
         config = YAML::load_file(file)
       rescue
-        raise MigrationError.new("Could not read defaults file #{file}: " + $!)
+        raise MigrationError.new("Could not read field map file #{file}: " + $!)
       end
 
+      # collect the class => path => header entries
+      config.each do |field, attr_list|
+        next if attr_list.blank?
+        # the header accessor method for the field
+        header = @loader.accessor(field)
+        raise MigrationError.new("Field defined in migration configuration #{file} not found in input file #{@input} headers: #{field}") if header.nil?
+        # associate each attribute path in the property value with the header
+        attr_list.split(/,\s*/).each do |path_s|
+          klass, path = create_attribute_path(path_s)
+          hash[klass][path] = header
+        end
+      end
+    end
+    
+    # Loads the defaults config files.
+    #
+    # @param [<String>, String] files the file or file array to load
+    # @return [<Class => <String => Object>>] the class => path => default value entries 
+    def load_defaults_files(files)
+      # collect the class => path => value entries from each defaults file
+      hash = LazyHash.new { Hash.new }
+      files.enumerate { |file| load_defaults_file(file, hash) }
+      hash
+    end
+    
+    # Loads the defaults config file into the given hash.
+    #
+    # @param [String] file the file to load
+    # @param [<Class => <String => Object>>] hash the class => path => default value entries 
+    def load_defaults_file(file, hash)
+      begin
+        config = YAML::load_file(file)
+      rescue
+        raise MigrationError.new("Could not read defaults file #{file}: " + $!)
+      end
       # collect the class => path => value entries
-      map = LazyHash.new { Hash.new }
       config.each do |path_s, value|
         next if value.nil_or_empty?
         klass, path = create_attribute_path(path_s)
-        map[klass][path] = value
+        hash[klass][path] = value
+      end
+    end    
+    # Loads the filter config files.
+    #
+    # @param [<String>, String] files the file or file array to load
+    # @return [<Class => <String => Object>>] the class => path => default value entries 
+    def load_filter_files(files)
+      # collect the class => path => value entries from each defaults file
+      hash = {}
+      files.enumerate { |file| load_filter_file(file, hash) }
+      hash
+    end
+    
+    # Loads the filter config file into the given hash.
+    #
+    # @param [String] file the file to load
+    # @param [<Class => <String => <Object => Object>>>] hash the class => path => input value => caTissue value entries 
+    def load_filter_file(file, hash)
+      begin
+        config = YAML::load_file(file)
+      rescue
+        raise MigrationError.new("Could not read filter file #{file}: " + $!)
+      end
+      # collect the class => attribute => filter entries
+      config.each do |path_s, flt|
+        next if flt.nil_or_empty?
+        klass, path = create_attribute_path(path_s)
+        unless path.size == 1 then
+          raise MigrationError.new("Migration filter configuration path not supported: #{path_s}")
+        end
+        attr = klass.standard_attribute(path.first.to_sym)
+        flt_hash = hash[klass] ||= {}
+        flt_hash[attr] = flt
       end
-
-      map
     end
 
     # @param [String] path_s a period-delimited path string path_s in the form _class_(._attribute_)+