jinx-migrate 2.1.1

data/lib/jinx/migration/reader.rb

@@ -0,0 +1,16 @@
+module Jinx
+  module Migration
+    # A prototypical source reader which enumerates the input records.
+    module Reader
+      include Enumerable
+  
+      # @param [String] name the migration mapping source field name, e.g. +First Name+
+      # @return [Symbol] the record value accessor symbol, e.g. +:first_name+
+      def accessor(name); end
+  
+      # @yield [rec] migrate the source record
+      # @yieldparam [{Symbol => Object}] rec the source accessor => value record
+      def each; end
+    end
+  end
+end

data/lib/jinx/migration/version.rb

@@ -0,0 +1,5 @@
+module Jinx
+  module Migrate
+    VERSION = "2.1.1"
+  end
+end

data/spec/bad/bad_spec.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Model
+  RESULTS = File.dirname(__FILE__) + '/../../test/results'
+
+  describe 'Bad' do
+    # The rejects file.
+    bad = RESULTS + '/bad/rejects.csv'
+    
+    # Migrate the input.
+    migrated = Jinx::Migrator.new(:debug => true, :target => Parent, :bad => bad,
+      :mapping => File.expand_path('fields.yaml', File.dirname(__FILE__)),
+      :shims => File.expand_path('shims.rb', File.dirname(__FILE__)),
+      :input => File.expand_path('parents.csv', File.dirname(__FILE__))
+    ).to_a
+    
+    # Validate the migration.
+    it "should migrate one record" do
+      migrated.size.should be 1
+    end
+    it "should capture two bad records" do
+      File.open(bad).to_a.size.should be 2
+    end
+  end
+end

data/spec/bad/fields.yaml

@@ -0,0 +1 @@
+Name: Parent.name

data/spec/bad/parents.csv

@@ -0,0 +1 @@
+Name

data/spec/bad/shims.rb

@@ -0,0 +1,16 @@
+module Model
+  shims Parent
+  
+  class Parent
+    # Simulate an error.
+    def migrate_name(value, row)
+      raise StandardError.new("Simulated error") if value == 'Mark'
+      value
+    end
+    
+    # Simulate invalidation.
+    def migration_valid?
+      name == 'Tom'
+    end
+  end
+end

data/spec/csv/join/join_helper.rb

@@ -0,0 +1,35 @@
+require 'spec/spec_helper'
+require 'fileutils'
+require 'jinx/csv/csvio'
+
+SOURCE = File.expand_path('source.csv', File.dirname(__FILE__))
+
+TARGET = File.expand_path('target.csv', File.dirname(__FILE__))
+
+RESULTS = File.dirname(__FILE__) + '/../../../test/results/join'
+
+OUTPUT = File.expand_path('output.csv', RESULTS)
+
+module Jinx
+  module JoinHelper
+    # Joins the given source fixture to the target fixture on the specified fields.
+    #
+    # @param [Symbol] source the source file fixture in the join spec directory
+    # @param [Symbol] target the target file fixture in the join spec directory
+    # @param [<String>] fields the source fields (default is all source fields)
+    # @return [<<String>>] the output records
+    def join(source, target, *fields, &block)
+      FileUtils.rm_rf OUTPUT
+      sf = File.expand_path("#{source}.csv", File.dirname(__FILE__))
+      tf = File.expand_path("#{target}.csv", File.dirname(__FILE__))
+      Jinx::CsvIO.join(sf, :to => tf, :for => fields, :as => OUTPUT, &block)
+      if File.exists?(OUTPUT) then
+        File.readlines(OUTPUT).map do |line|
+          line.chomp.split(',').map { |s| s unless s.blank? }
+        end
+      else
+        Array::EMPTY_ARRAY
+      end
+    end
+  end
+end

data/spec/csv/join/join_spec.rb

@@ -0,0 +1,100 @@
+require 'spec/csv/join/join_helper'
+
+shared_examples 'a join for all source fields' do 
+  it 'joins each record' do
+    @output.size.should be 10
+  end
+
+  it 'writes the output header row' do
+    @output.first.should == ['A', 'B', 'U', 'X']
+  end
+
+  it 'writes the matching source and target' do
+    @output[1].should == ['a1', 'b1', 'u', 'x']
+    @output[2].should == ['a1', 'b1', 'v', 'x']
+    @output[3].should == ['a1', 'b2', 'u', 'x']
+    @output[4].should == ['a1', 'b2', 'u', 'y']
+    @output[5].should == ['a2', 'b3', 'u', 'x']
+  end
+
+  it 'writes the unmatched source' do
+    # Note that String split truncates the trailing blank array items,
+    # so the comparison is to ['a2', 'b4', 'u'] rather than ['a2', 'b4', 'u', nil].
+    @output[6].should == ['a2', 'b4', 'u']
+    @output[9].should == ['a4', 'b7', 'u']
+  end
+
+  it 'writes the unmatched target' do
+    @output[7].should == ['a2', 'b5', nil, 'x']
+    @output[8].should == ['a3', nil, nil, 'x']
+  end
+end                               
+
+describe 'Join' do
+  include Jinx::JoinHelper
+  
+  context 'Join for all source fields' do
+    before(:all) { @output = join(:source, :target) }
+    
+    it_behaves_like 'a join for all source fields'
+  end
+  
+  context 'Join with block' do
+    before(:all) do
+      @output = join(:source, :target) do |rec|
+        curr = rec[0..1]
+        if curr == @prev then
+          rec[1] = nil
+        else 
+          @prev = curr
+        end
+        rec unless curr == ['a2', 'b3']
+      end
+    end
+
+    it 'preserves the output header row' do
+      @output.first.should == ['A', 'B', 'U', 'X']
+    end
+
+    it 'applies the block to the records before writing them to the ouput' do
+      @output[1].should == ['a1', 'b1', 'u', 'x']
+      @output[2].should == ['a1', nil, 'v', 'x']
+      @output[3].should == ['a1', 'b2', 'u', 'x']
+      @output[4].should == ['a1', nil, 'u', 'y']
+    end
+
+    it 'omits the record if the block returns nil' do
+      @output[5].should == ['a2', 'b4', 'u']
+    end
+  end
+  
+  context 'Join for jumbled source and target fields' do
+    before(:all) { @output = join(:jumbled_src, :jumbled_tgt) }
+    
+    it_behaves_like 'a join for all source fields'
+  end
+  
+  context 'Join for only the key source fields' do
+    before(:all) { @output = join(:source, :target, 'A', 'B') }
+
+    it 'joins each record' do
+      @output.size.should be 10
+    end
+
+    it 'writes the output header row' do
+      @output.first.should == ['A', 'B', 'X']
+    end
+
+    it 'writes the matching source and target records without the source-specific fields' do
+      @output[1].should == ['a1', 'b1', 'x']
+      @output[2].should == ['a1', 'b1', 'x']
+      @output[3].should == ['a1', 'b2', 'x']
+      @output[4].should == ['a1', 'b2', 'y']
+      @output[5].should == ['a2', 'b3', 'x']
+      @output[6].should == ['a2', 'b4']
+      @output[7].should == ['a2', 'b5', 'x']
+      @output[8].should == ['a3', nil, 'x']
+      @output[9].should == ['a4', 'b7']
+    end
+  end
+end

data/spec/csv/join/jumbled_src.csv

@@ -0,0 +1,7 @@
+A,U,B
+a1,u,b1
+a1,v,b1
+a1,u,b2
+a2,u,b3
+a2,u,b4
+a4,u,b7

data/spec/csv/join/jumbled_tgt.csv

@@ -0,0 +1,7 @@
+X,B,A
+x,b1,a1
+x,b2,a1
+y,b2,a1
+x,b3,a2
+x,b5,a2
+x,,a3

data/spec/csv/join/source.csv

@@ -0,0 +1,7 @@
+A,B,U
+a1,b1,u
+a1,b1,v
+a1,b2,u
+a2,b3,u
+a2,b4,u
+a4,b7,u

data/spec/csv/join/target.csv

@@ -0,0 +1,7 @@
+A,B,X
+a1,b1,x
+a1,b2,x
+a1,b2,y
+a2,b3,x
+a2,b5,x
+a3,,x

data/spec/extract/extract.rb

@@ -0,0 +1,13 @@
+module Model
+  shims Parent
+  
+  class Parent
+    @@id = 1
+    
+    def extract(io)
+      io << [name, @@id]
+      @@id += 1
+    end
+  end
+end
+  

data/spec/extract/extract_spec.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Model
+  describe 'Extract' do
+    EXTRACT = File.expand_path('ids.csv', Migration::Test::RESULTS + '/extract')
+    
+    HEADERS = ['Name', 'Id']
+    
+    before(:all) do
+      # Migrate the input.
+      @migrated = Jinx::Migrator.new(
+        :debug => true,
+        :target => Parent,
+        :mapping => File.expand_path('fields.yaml', File.dirname(__FILE__)),
+        :extract => EXTRACT,
+        :extract_headers => HEADERS,
+        :shims => File.expand_path('extract.rb', File.dirname(__FILE__)),
+        :input => File.expand_path('parents.csv', File.dirname(__FILE__))
+      ).to_a
+    end
+    
+    it "should migrate the records" do
+      @migrated.size.should be 3
+    end
+    
+    it "should create the extract" do
+      xtr = File.readlines(EXTRACT).map { |line| line.chomp }
+      xtr.size.should be 4
+      xtr[0].should == HEADERS.join(',')
+      1.upto(3) { |i| xtr[i].chomp.split(',').should == [@migrated[i - 1].name, i.to_s] }
+    end
+  end
+end

data/spec/extract/fields.yaml

@@ -0,0 +1 @@
+Name: Parent.name

data/spec/extract/parents.csv

@@ -0,0 +1 @@
+Name

data/spec/family/child_spec.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + '/family'
+
+module Family
+  describe Child do
+    before(:all) do
+      # Migrate the input.
+      @migrated = Jinx::Migrator.new(
+        :debug => true,
+        :target => Child,
+        :mapping => CONFIGS + '/children/fields.yaml',
+        :shims => SHIMS,
+        :input => DATA + '/children.csv'
+      ).to_a
+    end
+    
+    # Validate the migration.
+    it "should migrate the records" do
+      @migrated.size.should be 3
+    end
+    
+    it "should migrate the parents" do
+      @migrated.each do |child|
+        child.parents.size.should be 1
+      end
+    end
+  end
+end

data/spec/family/family.rb

@@ -0,0 +1,13 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+# Load the jinx family example.
+require Bundler.environment.specs.detect { |s| s.name == 'jinx' }.full_gem_path + '/examples/family/lib/family'
+
+module Family
+  include Jinx::Migratable
+  
+  ROOT = File.dirname(__FILE__) + '/../../examples/family'
+  DATA = ROOT + '/data'
+  CONFIGS = ROOT + '/conf'
+  SHIMS = ROOT + '/lib/shims.rb'
+end

data/spec/family/parent_spec.rb

@@ -0,0 +1,57 @@
+require File.dirname(__FILE__) + '/family'
+
+module Family
+  # The specification for the family example.
+  describe Parent do
+    before(:all) do
+      # Migrate the input.
+      @migrated = Jinx::Migrator.new(
+        :debug => true,
+        :target => Parent,
+        :mapping => CONFIGS + '/parents/fields.yaml',
+        :defaults => CONFIGS + '/parents/defaults.yaml',
+        :filters => CONFIGS + '/parents/values.yaml',
+        :shims => SHIMS,
+        :input => DATA + '/parents.csv'
+      ).to_a
+    end
+    
+    # Validate the migration.
+    it "should migrate the records" do
+      @migrated.size.should be 2
+    end
+    
+    it "should create a household" do
+      @migrated.each do |parent|
+        parent.household.should_not be nil
+      end
+    end
+    
+    it "should migrate the addresses" do
+      @migrated.each do |parent|
+        parent.household.address.should_not be nil
+      end
+    end
+    
+    it "should abbreviate the street" do
+      addr = @migrated.first.household.address
+      addr.street1.should match /St/
+      addr.street1.should_not match /Street/
+    end
+    
+    it "should add the default state" do
+      @migrated.each do |parent|
+        parent.household.address.state.should == 'IL'
+      end
+    end
+    
+    it "should migrate the spouse" do
+      @migrated.first.spouse.should_not be nil
+    end
+    
+    it "should set the spouse household" do
+      hh = @migrated.first.household
+      @migrated.first.spouse.household.should be hh
+    end
+  end
+end

data/spec/filter/fields.yaml

@@ -0,0 +1 @@
+Name: Parent.name

data/spec/filter/filter_spec.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+module Model
+  describe 'Filter' do
+    before(:all) do
+      @migrated = Jinx::Migrator.new(:debug => true, :target => Parent,
+        :mapping => File.expand_path('fields.yaml', File.dirname(__FILE__)),
+        :filters => File.expand_path('values.yaml', File.dirname(__FILE__)),
+        :input => File.expand_path('parents.csv', File.dirname(__FILE__))
+      ).to_a
+    end
+    
+    it "should filter the name" do
+      @migrated.size.should be 3
+      @migrated[0].name.should == 'Joseph'
+      @migrated[1].name.should == 'Christine'
+      @migrated[2].name.should == 'Other'
+    end
+  end
+end

data/spec/filter/parents.csv

@@ -0,0 +1 @@
+Name

data/spec/filter/values.yaml

@@ -0,0 +1,4 @@
+Parent.name:
+  Joe : Joseph
+  /Chris/i : Christine
+  /.*/ : Other

data/spec/primitive/children.csv

@@ -0,0 +1 @@
+Name,Flag,Cardinal,Decimal