jinx-migrate 2.1.1

data/lib/jinx/csv/joiner.rb

@@ -0,0 +1,196 @@
+require 'set'
+
+module Jinx
+  module Csv
+    # Merges two CSV files on common fields.
+    class Joiner
+      include Enumerable
+  
+      # @param [String, IO] source the join source
+      # @param [String, IO] target the join target (default stdin)
+      # @param [String, IO, nil] output the output file name or device (default stdout)
+      def initialize(source, target=nil, output=nil)
+        @source = source
+        @target = target || STDIN
+        @output = output || STDOUT
+      end
+  
+      # Joins the source to the target and writes the output. The source fields used are
+      # given by the +fields+ argument, if given. By default, all source fields are used.
+      #
+      # The output fields consist of the qualified source fields and all target fields.
+      # The output fields are in the following order:
+      # 1. The common fields, in order of occurrence in the source file.
+      # 2. The qualified source-specific fields, in order of occurrence in the source file. 
+      # 3. The target-specific fields, in order of occurrence in the target file. 
+      #
+      # The match is on the common qualified source and target fields.
+      # Both files must be sorted in order of the common fields, sequenced by their
+      # occurence in the source header.
+      #
+      # If an output argument is given, then the joined record is written to the output.
+      # If a block is given, then the block is called on each record prior to writing
+      # the record to the output. If the block returns nil, then the record is not
+      # written.
+      #
+      # @param [<String>] fields the optional source fields to merge
+      #   (default is all source fields)
+      # @yield [rec] process the output record and return the record to write
+      # @yieldparam [FasterCSV::Record] rec the output record
+      def join(*fields, &block)
+        CsvIO.open(@target) do |tgt|
+          CsvIO.open(@source) do |src|
+            # all source fields (unordered)
+            usflds = src.field_names.to_set
+            fields.each do |fld|
+              unless usflds.include?(fld) then
+                raise ArgumentError.new("CSV join field #{fld} not found in the source file #{@source}.")
+              end
+            end
+            # the qualified source fields (ordered)
+            qsflds = fields.empty? ? src.field_names : fields
+            tflds = tgt.field_names
+            @common = qsflds & tflds
+            # The headers consist of the common fields followed by the qualified
+            # source-specific fields followed by the target-specific fields.
+            hdrs = @common | qsflds | tflds
+            CsvIO.open(@output, :mode => 'w', :headers => hdrs) do |out|
+              merge(src, tgt, out, &block)
+            end
+          end
+        end
+    
+        alias :each :join
+      end
+  
+      private
+    
+      Buffer ||= Struct.new(:key, :record, :lookahead)
+      
+      # Merges the given source into the target as the output.
+      # The output headers must be in the order specified by {#join}.
+      #
+      # @param [CsvIO] source the source CSV IO
+      # @param [CsvIO] target the target CSV IO
+      # @param [CsvIO] output the merged output CSV IO
+      # @yield (see #join)
+      # @yieldparam (see #join)
+      # @see #join
+      def merge(source, target, output)
+        # the qualified source field accessors
+        sflds = source.accessors & output.accessors
+        # the target field accessors
+        tflds = target.accessors
+        # the common fields
+        @common = sflds & tflds
+        # The target-specific accessors
+        trest = tflds - @common
+        # The source-specific accessors
+        srest = output.accessors - trest - @common
+        # The output record
+        obuf = Array.new(output.accessors.size)
+        # The source/target current/next (key, record) buffers
+        # Read the first and second records into the buffers
+        sbuf = shift(source)
+        tbuf = shift(target)
+        # Compare the source and target.
+        while cmp = compare(sbuf, tbuf) do
+          # Fill the output record in three sections: the common, source and target fields.
+          obuf.fill do |i|
+            if i < @common.size then
+              cmp <= 0 ? sbuf.key[i] : tbuf.key[i]
+            elsif i < sflds.size then
+              # Only fill the output record with source values if there is a current source
+              # record and the target does not precede the source.
+              sbuf.record[srest[i - @common.size]] if sbuf and cmp <= 0
+            elsif tbuf and cmp >= 0
+              # Only fill the output record with target values if there is a current target
+              # record and the source does not precede the target.
+              tbuf.record[trest[i - sflds.size]]
+            end
+          end
+          orec = block_given? ? yield(obuf) : obuf 
+          # Emit the output record.
+          output << orec if orec
+          # Shift the buffers as necessary.
+          ss, ts = shift?(sbuf, tbuf, cmp), shift?(tbuf, sbuf, -cmp)
+          sbuf = shift(source, sbuf) if ss
+          tbuf = shift(target, tbuf) if ts
+        end
+      end
+    
+      # Returns whether to shift the given buffer as follows:
+      # * If the buffer precedes the other buffer, then true.
+      # * If the buffer succeeds the other buffer, then false.
+      # * Otherwise, if the lookahead record has the same key as the buffer record then true.
+      # * Otherwise, if the other lookahead record has a different key than the other record, then true.
+      #
+      # @param [Buffer] buf the record buffer to check
+      # @param [Buffer] other the other record buffer 
+      # @param [-1, 0, 1] order the buffer comparison 
+      # @return [Boolean] whether to shift the buffer
+      def shift?(buf, other, order)
+        case order
+        when -1 then
+          true
+        when 1 then
+          false
+        when 0 then
+          compare(buf, buf.lookahead) == 0 or compare(other, other.lookahead) != 0
+        end
+      end
+    
+      # Reads a record into the given buffers.
+      #
+      # @param [CsvIO] the open CSV stream to read
+      # @param [Buffer, nil] cbuf the current record buffer
+      # @return [Buffer, nil] the next current buffer, or nil if end of file 
+      def shift(csvio, buf=nil)
+        if buf then
+          return if buf.lookahead.nil?
+        else
+          # prime the look-ahead
+          buf = Buffer.new(nil, nil, look_ahead(csvio))
+          return shift(csvio, buf)
+        end
+        buf.record = buf.lookahead.record
+        buf.key = buf.lookahead.key
+        buf.lookahead = look_ahead(csvio, buf.lookahead)
+        buf
+      end
+    
+      # @param [CsvIO] csvio the CSV file stream
+      # @param [Buffer, nil] the look-ahead buffer
+      # @return [Buffer, nil] the modified look-ahead, or nil if end of file
+      def look_ahead(csvio, buf=nil)
+        rec = csvio.next || return
+        buf ||= Buffer.new
+        buf.record = rec
+        buf.key = @common.map { |k| rec[k] }
+        buf
+      end
+    
+      # Compares the given source and target buffers with result as follows:
+      # * If source and target are nil, then nil
+      # * If source is nil and target is not nil, then -1
+      # * If target is nil and source is not nil, then 1
+      # * Otherwise, the pair-wise comparison of the source and target keys
+      #
+      # @param [:key] the key holder
+      # @return [-1, 0 , 1, nil] the comparison result 
+      def compare(source, target)
+        return target.nil? ? nil : 1 if source.nil?
+        return -1 if target.nil?
+        source.key.each_with_index do |v1, i|
+          v2 = target.key[i]
+          next if v1.nil? and v2.nil?
+          return -1 if v1.nil?
+          return 1 if v2.nil?
+          cmp = v1 <=> v2
+          return cmp unless cmp == 0
+        end
+        0
+      end
+    end
+  end
+end

data/lib/jinx/migration/filter.rb

@@ -0,0 +1,167 @@
+require 'jinx/helpers/validation'
+
+module Jinx
+  module Migration
+    # Transforms input values to a result based on a migration filter configuration.
+    # Each configuration entry is one of the following:
+    #   * literal: literal
+    #   * regexp: literal
+    #   * regexp: template
+    #
+    # The regexp template can include match references (+$1+, +$2+, etc.) corresponding to the regexp captures.
+    # If the input value equals a literal, then the mapped literal is returned. Otherwise, if the input value
+    # matches a regexp, then the mapped transformation is returned after reference substitution. Otherwise,
+    # the input value is returned unchanged.
+    #
+    # For example, the config:
+    #   /(\d{1,2})\/x\/(\d{1,2})/ : $1/15/$2
+    #   n/a : ~
+    # converts the input value as follows:
+    #   3/12/02 => 3/12/02 (no match)
+    #   5/x/04 => 5/15/04
+    #   n/a => nil
+    #
+    # A catch-all +/.*/+ regexp transforms any value which does not match another value or regexp, e.g.:
+    #   /^(\d+(\.\d*)?)( g(ram)?s?)?$/ : $1 
+    #   /.*/ : 0
+    # converts the input value as follows:
+    #   3 => 3
+    #   4.3 grams => 4.3
+    #   unknown => 0
+    class Filter
+      # Builds the filter proc from the given specification or block.
+      # If both a specification and a block are given, then the block is applied before
+      # the specificiation.  
+      #
+      # @param [String] spec the filter configuration specification.
+      # @yield [value] converts the input field value into a caTissue property value
+      # @yieldparam value the CSV input value 
+      def initialize(spec=nil, &block)
+        @proc = spec ? to_proc(spec, &block) : block
+        raise ArgumentError.new("Migration filter is missing both a specification and a block") if @proc.nil?
+      end
+        
+      # @param [String] value the input string
+      # @return the transformed result
+      def transform(value)
+        @proc.call(value)
+      end
+    
+      private
+    
+      # The pattern to match a regular expression with captures.
+      # @private
+      REGEXP_PAT = /^\/(.*[^\\])\/([inx]+)?$/
+      
+      # Builds the filter proc from the given specification.  
+      # If both a specification and a block are given, then the block is applied before
+      # the specificiation.  
+      #
+      # @param (see #initialize)
+      # @yield (see #initialize)
+      # @yieldparam (see #initialize)
+      # @return [Proc] a proc which convert the input field value into a caTissue property value
+      def to_proc(spec=nil, &block)
+        # Split the filter spec into a straight value => value hash and a pattern => value hash.
+        ph, vh = spec.split { |k, v| k =~ REGEXP_PAT }
+        # The Regexp => value hash is built from the pattern => value hash.
+        reh = regexp_hash(ph)
+        # The value proc.
+        value_proc(reh, vh)
+      end
+      
+      # @param {Regexp => (Object, <Integer>)} regexp_hash the regexp => (result, indexes) hash
+      # @param {String => Object} value_hash the value => result hash
+      # @yield (see #to_proc)
+      # @yieldparam (see #to_proc)
+      # @return [Proc] a proc which convert the input field value into a caTissue property value
+      def value_proc(regexp_hash, value_hash)
+        # The new proc matches preferentially on the literal value, then the first matching regexp.
+        # If no match on either a literal or a regexp, then the value is preserved.
+        Proc.new do |value|
+          value = yield(value) if block_given?
+          if value_hash.has_key?(value) then
+            value_hash[value]
+          else
+            # The first regex which matches the value.
+            regexp = regexp_hash.detect_key { |re| value =~ re }
+            # If there is a match, then apply the filter to the match data.
+            # Otherwise, pass the value through unmodified.
+            if regexp then
+              reval, ndxs = regexp_hash[regexp]
+              if ndxs.empty? or not String === reval then
+                reval
+              else
+                # The match captures (cpts[i - 1] is $i match).
+                cpts = $~.captures
+                # Substitute the capture index specified in the configuration for the corresponding
+                # template variable, e.g. the value filter:
+                #   /(Grade )?(\d)/ : $2
+                # is parsed as (reval, ndxs) = (/(Grade )?(\d)/, 1) 
+                # and transforms 'Grade 3' to cpts[0], or '3'.
+                fmtd = reval % ndxs.map { |i| cpts[i] }
+                fmtd unless fmtd.blank?
+              end
+            elsif defined? @catch_all then
+              @catch_all
+            else
+              value
+            end
+          end
+        end
+      end
+      
+      # Parses the configuration pattern string => value hash into a regexp => value hash
+      # qualified by the match indexes used to substitute match captures into the hash value.
+      #
+      # The pattern hash value can include match references ($1, $2, etc.). In that case,
+      # the match captures substitute into a %s format reference in the result.
+      #
+      # @example
+      #   regexp_hash({'/Golf/i' => 1}) #=> {1, []}
+      #   regexp_hash({'/Hole (\d{1,2})/' => $1}) #=> {'%', [0]}
+      #                        
+      # @param [{String => Object}] pat_hash the string => value hash
+      # @return [{Regexp => (Object, <Integer>)}] the corresponding regexp => (value, indexes) hash
+      def regexp_hash(pat_hash)
+        # The Regexp => value hash is built from the pattern => value hash.
+        reh = {}
+        # Make a matcher for each regexp pattern.
+        pat_hash.each do |k, v|
+          # The /pattern/opts string is parsed to the pattern and options.
+          pat, opt = REGEXP_PAT.match(k).captures
+          # the catch-all matcher
+          if pat == '.*' then
+            @catch_all = v
+            next
+          end
+          # Convert the regexp i option character to a Regexp initializer parameter.
+          reopt = if opt then
+            case opt
+              when 'i' then Regexp::IGNORECASE
+              else Jinx.fail(MigrationError, "Migration value filter regular expression #{k} qualifier not supported: expected 'i', found '#{opt}'")
+            end
+          end
+          # the Regexp object
+          re = Regexp.new(pat, reopt)
+          # Replace each $ match reference with a %s format reference.
+          reh[re] = parse_regexp_value(v)
+        end
+        reh
+      end
+
+      # @example
+      #   parse_regexp_value('Grade $2') #=> ['Grade %s', [1]]
+      # @param value the value in the configuration regexp => value entry
+      # @return (Object, <Integer>) the parsed (value, indexes) 
+      # @see #regexp_hash
+      def parse_regexp_value(value)
+        return [value, Array::EMPTY_ARRAY] unless value =~ /\$\d/
+        tmpl = value.gsub(/\$\d/, '%s')
+        # Look for match references of the form $n.
+        ndxs = value.scan(/\$(\d)/).map { |matches| matches.first.to_i - 1 }
+        [tmpl, ndxs]
+      end
+    end
+  end
+end

data/lib/jinx/migration/migratable.rb

@@ -0,0 +1,244 @@
+module Jinx
+  # The Migratable mix-in adds migration support for Resource domain objects.
+  # For each migration Resource created by a Migrator, the migration process
+  # is as follows:
+  #
+  # 1. The migrator creates the Resource using the empty constructor.
+  #
+  # 2. Each input field value which maps to a Resource attribute is obtained from the
+  #    migration source.
+  #
+  # 3. If the Resource class implements a method +migrate_+_attribute_ for the
+  #    migration _attribute_, then that migrate method is called with the input value
+  #    argument. If there is a migrate method, then the attribute is set to the
+  #    result of calling that method, otherwise the attribute is set to the original
+  #    input value.
+  #
+  #    For example, if the +Name+ input field maps to +Parent.name+, then a
+  #    custom +Parent+ +migrate_name+ shim method can be defined to reformat
+  #    the input name.
+  #
+  # 4. The Resource attribute is set to the (possibly modified) value.
+  #
+  # 5. After all input fields are processed, then {#migration_valid?} is called to
+  #    determine whether the migrated object can be used. {#migration_valid?} is true
+  #    by default, but a migration shim can add a validation check,
+  #    migrated Resource class to return false for special cases.
+  #
+  #    For example, a custom +Parent+ +migration_valid?+ shim method can be
+  #    defined to return whether there is a non-empty input field value.
+  #
+  # 6. After the migrated objects are validated, then the Migrator fills in
+  #    dependency hierarchy gaps. For example, if the Resource class +Parent+
+  #    owns the +household+ dependent which in turn owns the +address+ dependent
+  #    and the migration has created a +Parent+ and an +Address+ but no +Household+,
+  #    then an empty +Household+ is created which is owned by the migrated +Parent+
+  #    and owns the migrated +Address+.
+  #
+  # 7. After all dependencies are filled in, then the independent references are set
+  #    for each created Resource (including the new dependents). If a created
+  #    Resource has an independent non-collection Resource reference attribute
+  #    and there is a migrated instance of that attribute type, then the attribute
+  #    is set to that migrated instance.
+  #
+  #    For example, if +Household+ has a +address+ attribute and there is a
+  #    single migrated +Address+ instance, then the +address+ attribute is set
+  #    to that migrated +Address+ instance.
+  #
+  #    If the referencing class implements a method +migrate_+_attribute_ for the
+  #    migration _attribute_, then that migrate method is called with the referenced
+  #    instance argument. The result is used to set the attribute. Otherwise, the
+  #    attribute is set to the original referenced instance.
+  #
+  #    There must be a single unambiguous candidate independent instance, e.g. in the
+  #    unlikely but conceivable case that two +Address+ instances are migrated, then the
+  #    +address+ attribute is not set. Similarly, collection attributes are not set,
+  #    e.g. a +Address+ +protocols+ attribute is not set to a migrated +Protocol+
+  #    instance.
+  #
+  # 8. The {#migrate} method is called to complete the migration. As described in the
+  #    method documentation, a migration shim Resource subclass can override the
+  #    method for custom migration processing, e.g. to migrate the ambiguous or
+  #    collection attributes mentioned above, or to fill in missing values.
+  #
+  #    Note that there is an extensive set of attribute defaults defined in the +Jinx::Resource+
+  #    application domain classes. These defaults are applied in a migration database save
+  #    action and need not be set in a migration shim. For example, if an acceptable
+  #    default for an +Address.country+ property is defined in the +Address+ meta-data,
+  #    then the country does not need to be set in a migration shim.
+  module Migratable
+    # Completes setting this Migratable domain object's attributes from the given input row.
+    # This method is responsible for migrating attributes which are not mapped
+    # in the configuration. It is called after the configuration attributes for
+    # the given row are migrated and before {#migrate_references}.
+    #
+    # This base implementation is a no-op.
+    # Subclasses can modify this method to complete the migration. The overridden
+    # methods should call +super+ to pick up the superclass migration.
+    #
+    # @param [{Symbol => Object}] row the input row field => value hash
+    # @param [<Resource>] migrated the migrated instances, including this domain object
+    def migrate(row, migrated)
+    end
+
+    # Returns whether this migration target domain object is valid. The default is true.
+    # A migration shim should override this method on the target if there are conditions
+    # which determine whether the migration should be skipped for this target object.
+    #
+    # @return [Boolean] whether this migration target domain object is valid
+    def migration_valid?
+      true
+    end
+
+    # Migrates this domain object's migratable references. This method is called by the
+    # Migrator and should not be overridden by subclasses. Subclasses tailor
+    # individual reference attribute migration by defining a +migrate_+_attribute_ method
+    # for the _attribute_ to modify.
+    #
+    # The migratable reference attributes consist of the non-collection saved independent
+    # attributes and the unidirectional dependent attributes which don't already have a value.
+    # For each such migratable attribute, if there is a single instance of the attribute
+    # type in the given migrated domain objects, then the attribute is set to that
+    # migrated instance.
+    #
+    # If the attribute is associated with a method in proc_hash, then that method is called
+    # on the migrated instance and input row. The attribute is set to the method return value.
+    # proc_hash includes an entry for each +migrate_+_attribute_ method defined by this
+    # Resource's class.
+    #
+    # @param [{Symbol => Object}] row the input row field => value hash
+    # @param [<Resource>] migrated the migrated instances, including this Resource
+    # @param [Class] target the migration target class
+    # @param [{Symbol => Proc}, nil] proc_hash a hash that associates this domain object's
+    #   attributes to a migration shim block
+    def migrate_references(row, migrated, target, proc_hash=nil)
+      # migrate the owner
+      migratable__migrate_owner(row, migrated, target, proc_hash)
+      # migrate the remaining attributes
+      migratable__set_nonowner_references(migratable_independent_attributes, row, migrated, proc_hash)
+      migratable__set_nonowner_references(self.class.unidirectional_dependent_attributes, row, migrated, proc_hash)
+    end
+                 
+    # Returns this Resource's class {Propertied#independent_attributes}.
+    # Applications can override this implement to restrict the independent attributes which
+    # are migrated, e.g. to include only saved independent attributes. 
+    #
+    # @return the attributes to migrate
+    def migratable_independent_attributes
+      self.class.independent_attributes
+    end
+    
+    # Extracts the content of this migration target to the given file.
+    #
+    # This base implementation is a no-op.
+    # Subclasses can modify this method to write data to the extract.
+    #
+    # @param [IO] file the extract output stream 
+    def extract(file)
+    end
+    
+    private
+    
+    # Migrates the owner as follows:
+    # * If there is exactly one migrated owner, then the owner reference is
+    #   set to that owner.
+    # * Otherwise, if there is more than one owner but only one owner instance
+    #   of the given target class, then that target instance is that owner.
+    # * Otherwise, no reference is set.
+    #
+    # @param row (see #migrate_references)
+    # @param migrated (see #migrate_references)
+    # @param target (see #migrate_references)
+    # @param proc_hash (see #migrate_references)
+    # @return [Resource, nil] the migrated owner, if any
+    def migratable__migrate_owner(row, migrated, target, proc_hash=nil)
+      # the owner attributes=> migrated reference hash
+      ovh = self.class.owner_attributes.to_compact_hash do |mattr|
+        pa = self.class.property(mattr)
+        migratable__target_value(pa, row, migrated, proc_hash)
+      end
+      # If there is more than one owner candidate, then select the owner
+      # attribute which references the target. If there is more than one
+      # such attribute, then select the preferred owner.
+      if ovh.size > 1 then
+        tvh = ovh.filter_on_value { |ov| target === ov }.to_hash
+        if tvh.size == 1 then
+          ovh = tvh
+        else
+          ownrs = ovh.values.uniq
+          if ownrs.size == 1 then
+            ovh = {ovh.keys.first => ownrs.first}
+          else
+            logger.debug { "The migrated dependent #{qp} has ambiguous migrated owner references #{ovh.qp}." }
+            preferred = migratable__preferred_owner(ownrs)
+            if preferred then
+              logger.debug { "The preferred dependent #{qp} migrated owner reference is #{preferred.qp}." }
+              ovh = {ovh.keys.detect { |k| ovh[k] == preferred } => preferred}
+            end
+          end
+        end
+      end
+      if ovh.size == 1 then
+        oattr, oref = ovh.first
+        set_property_value(oattr, oref)
+        logger.debug { "Set the #{qp} #{oattr} owner to the migrated #{oref.qp}." }
+      end
+      oref
+    end
+    
+    # This base implementation returns nil. Subclasses can override this to select a preferred owner.
+    #
+    # @param [<Resource>] candidates the migrated owners
+    # @return [Resource] the preferred owner
+    def migratable__preferred_owner(candidates)
+      nil
+    end
+    
+    # @param [Property::Filter] the attributes to set
+    # @param row (see #migrate_references)
+    # @param migrated (see #migrate_references)
+    # @param proc_hash (see #migrate_references)
+    def migratable__set_nonowner_references(attr_filter, row, migrated, proc_hash=nil)
+      attr_filter.each_pair do |mattr, pa|
+        # skip owners
+        next if pa.owner?
+        # the target value
+        ref = migratable__target_value(pa, row, migrated, proc_hash) || next
+        if pa.collection? then
+          # the current value
+          value = send(pa.reader) || next
+          value << ref
+          logger.debug { "Added the migrated #{ref.qp} to #{qp} #{mattr}." }
+        else 
+          current = send(mattr)
+          if current then
+            logger.debug { "Ignoring the migrated #{ref.qp} since #{qp} #{mattr} is already set to #{current.qp}." }
+          else
+            set_property_value(mattr, ref)
+            logger.debug { "Set the #{qp} #{mattr} to the migrated #{ref.qp}." }
+          end
+        end
+      end
+    end
+    
+    # @param [Property] pa the reference attribute
+    # @param row (see #migrate_references)
+    # @param migrated (see #migrate_references)
+    # @param proc_hash (see #migrate_references)
+    # @return [Resource, nil] the migrated instance of the given class, or nil if there is not
+    #   exactly one such instance
+    def migratable__target_value(pa, row, migrated, proc_hash=nil)
+      # the migrated references which are instances of the attribute type
+      refs = migrated.select { |other| other != self and pa.type === other }
+      # skip ambiguous references
+      if refs.size > 1 then logger.debug { "Migrator did not set references to ambiguous targets #{refs.pp_s}." } end
+      return unless refs.size == 1
+      # the single reference
+      ref = refs.first
+      # the shim method, if any
+      proc = proc_hash[pa.to_sym] if proc_hash
+      # if there is a shim method, then call it
+      proc ? proc.call(self, ref, row) : ref
+    end
+  end
+end